v5.2.7

add API headers on upload call

.github/FUNDING.yml

@@ -0,0 +1,12 @@
+# These are supported funding model platforms
+
+github: dbgate
+patreon: # Replace with a single Patreon username
+open_collective: dbgate
+ko_fi: # Replace with a single Ko-fi username
+tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
+community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
+liberapay: # Replace with a single Liberapay username
+issuehunt: # Replace with a single IssueHunt username
+otechie: # Replace with a single Otechie username
+custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2']

.github/ISSUE_TEMPLATE/bug_report.md

@@ -28,6 +28,3 @@ If applicable, add screenshots to help explain your problem.
  - Install source [e.g. installer/SNAP/Docker/NPM]
  - Type - Web/Application
  - Database engine: [e.g. MySQL/PostgreSQL/SQL Server]
-
-**Additional context**
-Anything else you think might be helpful

.github/ISSUE_TEMPLATE/feature_request.md

@@ -15,6 +15,3 @@ A clear and concise description of what you want to happen.
 
 **Describe alternatives you've considered**
 A clear and concise description of any alternative solutions or features you've considered.
-
-**Additional context**
-Add any other context or screenshots about the feature request here.

.github/workflows/build-app-beta.yaml

@@ -1,4 +1,4 @@
-name: Electron app
+name: Electron app BETA
 
 on:
   push:
@@ -10,22 +10,29 @@ jobs:
     runs-on: ${{ matrix.os }}
 
     strategy:
+      fail-fast: false
       matrix:
-        os: [ubuntu-18.04, windows-2016]
-        # os: [macOS-10.14, windows-2016, ubuntu-18.04]
+        os: [macos-12, windows-2022, ubuntu-22.04]
+        # os: [macOS-10.15]
 
     steps:
       - name: Context
         env:
           GITHUB_CONTEXT: ${{ toJson(github) }}
         run: echo "$GITHUB_CONTEXT"
-      - uses: actions/checkout@v1
+      - uses: actions/checkout@v2
         with:
           fetch-depth: 1
-      - name: Use Node.js 10.x
+      - name: Use Node.js 14.x
         uses: actions/setup-node@v1
         with:
-          node-version: 10.x
+          node-version: 14.x
+      - name: yarn adjustPackageJson
+        run: |
+          yarn adjustPackageJson
+      - name: yarn set timeout
+        run: |
+          yarn config set network-timeout 100000
       - name: yarn install
         run: |
           yarn install
@@ -35,39 +42,56 @@ jobs:
       - name: fillNativeModulesElectron
         run: |
           yarn fillNativeModulesElectron
+      - name: fillPackagedPlugins
+        run: |
+          yarn fillPackagedPlugins
       - name: Install Snapcraft
-        if: matrix.os == 'ubuntu-18.04'
+        if: matrix.os == 'ubuntu-22.04'
         uses: samuelmeuli/action-snapcraft@v1
       - name: Publish
         run: |
           yarn run build:app
         env:
           GH_TOKEN: ${{ secrets.GH_TOKEN }} # token for electron publish
-          WIN_CSC_LINK: ${{ secrets.WINCERT_CERTIFICATE }}
-          WIN_CSC_KEY_PASSWORD: ${{ secrets.WINCERT_PASSWORD }}
 
-      - name: Save snap login
-        if: matrix.os == 'ubuntu-18.04'
-        run: 'echo "$SNAPCRAFT_LOGIN" > snapcraft.login'
-        shell: bash
-        env:
-          SNAPCRAFT_LOGIN: ${{secrets.SNAPCRAFT_LOGIN}}
+          WIN_CSC_LINK: ${{ secrets.WINCERT_2025 }}
+          WIN_CSC_KEY_PASSWORD: ${{ secrets.WINCERT_2025_PASSWORD }}
+          # WIN_CSC_LINK: ${{ secrets.WINCERT_CERTIFICATE }}
+          # WIN_CSC_KEY_PASSWORD: ${{ secrets.WINCERT_PASSWORD }}
+
+          CSC_LINK: ${{ secrets.APPLECERT_CERTIFICATE }}
+          CSC_KEY_PASSWORD: ${{ secrets.APPLECERT_PASSWORD }}
+
+          APPLE_ID: ${{ secrets.APPLE_ID }}
+          APPLE_ID_PASSWORD: ${{ secrets.APPLE_ID_PASSWORD }}
 
       - name: publishSnap
-        if: matrix.os == 'ubuntu-18.04'
+        if: matrix.os == 'ubuntu-22.04'
         run: |
-          snapcraft login --with snapcraft.login
           snapcraft upload --release=beta app/dist/*.snap
+        env:
+          SNAPCRAFT_STORE_CREDENTIALS: ${{secrets.SNAPCRAFT_LOGIN}}
 
       - name: Copy artifacts
         run: |
           mkdir artifacts          
 
+          cp app/dist/*.deb artifacts/dbgate-beta.deb || true
+          cp app/dist/*x86*.AppImage artifacts/dbgate-beta.AppImage || true
+          cp app/dist/*arm64*.AppImage artifacts/dbgate-beta-arm64.AppImage || true
+          cp app/dist/*armv7l*.AppImage artifacts/dbgate-beta-armv7l.AppImage || true
+          cp app/dist/*win*.exe artifacts/dbgate-beta.exe || true
+          cp app/dist/*win_x64.zip artifacts/dbgate-windows-beta.zip || true
+          cp app/dist/*win_arm64.zip artifacts/dbgate-windows-beta-arm64.zip || true
+          cp app/dist/*-mac_x64.dmg artifacts/dbgate-beta.dmg || true
+          cp app/dist/*-mac_arm64.dmg artifacts/dbgate-beta-arm64.dmg || true
+
           mv app/dist/*.exe artifacts/ || true
+          mv app/dist/*.zip artifacts/ || true
           mv app/dist/*.AppImage artifacts/ || true
           mv app/dist/*.deb artifacts/ || true
           mv app/dist/*.snap artifacts/ || true
-#          mv app/dist/*.dmg artifacts/ || true
+          mv app/dist/*.dmg artifacts/ || true
 
       - name: Upload artifacts
         uses: actions/upload-artifact@v1

.github/workflows/build-app.yaml

@@ -14,24 +14,33 @@ jobs:
     runs-on: ${{ matrix.os }}
 
     strategy:
+      fail-fast: false
       matrix:
-        # os: [ubuntu-18.04, windows-2016]
-        os: [macOS-10.14, windows-2016, ubuntu-18.04]
+        # os: [ubuntu-22.04, windows-2016]
+        os: [macos-12, windows-2022, ubuntu-22.04]
 
     steps:
       - name: Context
         env:
           GITHUB_CONTEXT: ${{ toJson(github) }}
         run: echo "$GITHUB_CONTEXT"
-      - uses: actions/checkout@v1
+      - uses: actions/checkout@v2
         with:
           fetch-depth: 1
-      - name: Use Node.js 10.x
+      - name: Use Node.js 14.x
         uses: actions/setup-node@v1
         with:
-          node-version: 10.x
+          node-version: 14.x
+      - name: yarn adjustPackageJson
+        run: |
+          yarn adjustPackageJson
+      - name: yarn set timeout
+        run: |
+          yarn config set network-timeout 100000
       - name: yarn install
         run: |
+          # yarn --version
+          # yarn config set network-timeout 300000
           yarn install
       - name: setCurrentVersion
         run: |
@@ -39,44 +48,56 @@ jobs:
       - name: fillNativeModulesElectron
         run: |
           yarn fillNativeModulesElectron
+      - name: fillPackagedPlugins
+        run: |
+          yarn fillPackagedPlugins
       - name: Install Snapcraft
-        if: matrix.os == 'ubuntu-18.04'
+        if: matrix.os == 'ubuntu-22.04'
         uses: samuelmeuli/action-snapcraft@v1
       - name: Publish
         run: |
           yarn run build:app
         env:
           GH_TOKEN: ${{ secrets.GH_TOKEN }} # token for electron publish
-          WIN_CSC_LINK: ${{ secrets.WINCERT_CERTIFICATE }}
-          WIN_CSC_KEY_PASSWORD: ${{ secrets.WINCERT_PASSWORD }}
+
+          WIN_CSC_LINK: ${{ secrets.WINCERT_2025 }}
+          WIN_CSC_KEY_PASSWORD: ${{ secrets.WINCERT_2025_PASSWORD }}
+          # WIN_CSC_LINK: ${{ secrets.WINCERT_CERTIFICATE }}
+          # WIN_CSC_KEY_PASSWORD: ${{ secrets.WINCERT_PASSWORD }}
+
+          CSC_LINK: ${{ secrets.APPLECERT_CERTIFICATE }}
+          CSC_KEY_PASSWORD: ${{ secrets.APPLECERT_PASSWORD }}
+
+          APPLE_ID: ${{ secrets.APPLE_ID }}
+          APPLE_ID_PASSWORD: ${{ secrets.APPLE_ID_PASSWORD }}
 
       - name: generatePadFile
         run: |
           yarn generatePadFile
 
-      - name: Save snap login
-        if: matrix.os == 'ubuntu-18.04'
-        run: 'echo "$SNAPCRAFT_LOGIN" > snapcraft.login'
-        shell: bash
-        env:
-          SNAPCRAFT_LOGIN: ${{secrets.SNAPCRAFT_LOGIN}}
-
       - name: publishSnap
-        if: matrix.os == 'ubuntu-18.04'
+        if: matrix.os == 'ubuntu-22.04'
         run: |
-          snapcraft login --with snapcraft.login
           snapcraft upload --release=stable app/dist/*.snap
+        env:
+          SNAPCRAFT_STORE_CREDENTIALS: ${{secrets.SNAPCRAFT_LOGIN}}
 
       - name: Copy artifacts
         run: |
           mkdir artifacts          
 
           cp app/dist/*.deb artifacts/dbgate-latest.deb || true
-          cp app/dist/*.AppImage artifacts/dbgate-latest.AppImage || true
+          cp app/dist/*x86*.AppImage artifacts/dbgate-latest.AppImage || true
+          cp app/dist/*arm64*.AppImage artifacts/dbgate-latest-arm64.AppImage || true
+          cp app/dist/*armv7l*.AppImage artifacts/dbgate-latest-armv7l.AppImage || true
           cp app/dist/*.exe artifacts/dbgate-latest.exe || true
-          cp app/dist/*.dmg artifacts/dbgate-latest.dmg || true
+          cp app/dist/*win_x64.zip artifacts/dbgate-windows-latest.zip || true
+          cp app/dist/*win_arm64.zip artifacts/dbgate-windows-latest-arm64.zip || true
+          cp app/dist/*-mac_universal.dmg artifacts/dbgate-latest.dmg || true
+          cp app/dist/*-mac_x64.dmg artifacts/dbgate-latest-x64.dmg || true
 
           mv app/dist/*.exe artifacts/ || true
+          mv app/dist/*.zip artifacts/ || true
           mv app/dist/*.AppImage artifacts/ || true
           mv app/dist/*.deb artifacts/ || true
           mv app/dist/*.dmg artifacts/ || true
@@ -107,18 +128,18 @@ jobs:
       #     mv app/dist/latest.yml artifacts/latest.yml || true
 
       - name: Copy latest.yml (windows)
-        if: matrix.os == 'windows-2016'
+        if: matrix.os == 'windows-2022'
         run: |
           mv app/dist/latest.yml artifacts/latest.yml || true
           mv app/dist/dbgate-pad.xml artifacts/ || true
 
       - name: Copy latest-linux.yml
-        if: matrix.os == 'ubuntu-18.04'
+        if: matrix.os == 'ubuntu-22.04'
         run: |
           mv app/dist/latest-linux.yml artifacts/latest-linux.yml || true
 
       - name: Copy latest-mac.yml
-        if: matrix.os == 'macOS-10.14'
+        if: matrix.os == 'macos-12'
         run: |
           mv app/dist/latest-mac.yml artifacts/latest-mac.yml || true
 

.github/workflows/build-docker.yaml

@@ -1,17 +1,11 @@
 name: Docker image
 
-# on: [push]
-
 on:
   push:
     tags:
       - 'v[0-9]+.[0-9]+.[0-9]+'
-    # - 'v*' # Push events to matching v*, i.e. v1.0, v20.15.10
-
-# on:
-#   push:
-#     branches:
-#       - production
+      - 'v[0-9]+.[0-9]+.[0-9]+-beta.[0-9]+'
+      - 'v[0-9]+.[0-9]+.[0-9]+-docker.[0-9]+'
 
 jobs:
   build:
@@ -20,23 +14,53 @@ jobs:
 
     strategy:
       matrix:
-        os: [ubuntu-18.04]
-        # os: [macOS-10.14, windows-2016, ubuntu-18.04]
+        os: [ubuntu-22.04]
 
     steps:
       - name: Context
         env:
           GITHUB_CONTEXT: ${{ toJson(github) }}
         run: echo "$GITHUB_CONTEXT"
-      - uses: actions/checkout@v1
+      - uses: actions/checkout@v2
         with:
           fetch-depth: 1
-      - name: Use Node.js 10.x
+
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v4
+        with:
+          images: |
+            dbgate/dbgate
+          flavor: |
+            latest=false
+          tags: |
+            type=raw,value=beta,enable=${{ contains(github.ref_name, '-docker.') || contains(github.ref_name, '-beta.') }}
+
+            type=match,pattern=\d+.\d+.\d+,enable=${{ !contains(github.ref_name, '-docker.') && !contains(github.ref_name, '-beta.') }}
+            type=raw,value=latest,enable=${{ !contains(github.ref_name, '-docker.') && !contains(github.ref_name, '-beta.') }}
+
+      - name: Docker alpine meta
+        id: alpmeta
+        uses: docker/metadata-action@v4
+        with:
+          images: |
+            dbgate/dbgate
+          flavor: |
+            latest=false
+          tags: |
+            type=raw,value=beta-alpine,enable=${{ contains(github.ref_name, '-docker.') || contains(github.ref_name, '-beta.') }}
+
+            type=match,pattern=\d+.\d+.\d+,suffix=-alpine,enable=${{ !contains(github.ref_name, '-docker.') && !contains(github.ref_name, '-beta.') }}
+            type=raw,value=alpine,enable=${{ !contains(github.ref_name, '-docker.') && !contains(github.ref_name, '-beta.') }}
+
+      - name: Use Node.js 14.x
         uses: actions/setup-node@v1
         with:
-          node-version: 10.x
+          node-version: 14.x
       - name: yarn install
         run: |
+          # yarn --version
+          # yarn config set network-timeout 300000
           yarn install
       - name: setCurrentVersion
         run: |
@@ -44,11 +68,28 @@ jobs:
       - name: Prepare docker image
         run: |
           yarn run prepare:docker
-      - name: Build docker image
-        run: |
-          docker build ./docker -t dbgate
-      - name: Push docker image
-        run: |
-          docker tag dbgate dbgate/dbgate
-          docker login -u ${{ secrets.DOCKER_USERNAME }} -p ${{ secrets.DOCKER_PASSWORD }}
-          docker push dbgate/dbgate
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+      - name: Login to DockerHub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Build and push
+        uses: docker/build-push-action@v3
+        with:
+          push: true
+          context: ./docker
+          tags: ${{ steps.meta.outputs.tags }}
+          platforms: linux/amd64,linux/arm64,linux/arm/v7
+
+      - name: Build and push alpine
+        uses: docker/build-push-action@v3
+        with:
+          push: true
+          context: ./docker
+          file: ./docker/Dockerfile-alpine
+          tags: ${{ steps.alpmeta.outputs.tags }}
+          platforms: linux/amd64,linux/arm64,linux/arm/v7

.github/workflows/build-npm.yaml

@@ -6,7 +6,7 @@ on:
   push:
     tags:
       - 'v[0-9]+.[0-9]+.[0-9]+'
-      # - 'v[0-9]+.[0-9]+.[0-9]+-alpha.[0-9]+'
+      - 'v[0-9]+.[0-9]+.[0-9]+-alpha.[0-9]+'
 
 # on:
 #   push:
@@ -20,21 +20,20 @@ jobs:
 
     strategy:
       matrix:
-        os: [ubuntu-18.04]
-        # os: [macOS-10.14, windows-2016, ubuntu-18.04]
+        os: [ubuntu-22.04]
 
     steps:
       - name: Context
         env:
           GITHUB_CONTEXT: ${{ toJson(github) }}
         run: echo "$GITHUB_CONTEXT"
-      - uses: actions/checkout@v1
+      - uses: actions/checkout@v2
         with:
           fetch-depth: 1
-      - name: Use Node.js 10.x
+      - name: Use Node.js 14.x
         uses: actions/setup-node@v1
         with:
-          node-version: 10.x
+          node-version: 14.x
 
       - name: Configure NPM token
         env:
@@ -84,8 +83,68 @@ jobs:
         working-directory: packages/web
         run: |
           npm publish
-                                        
-      - name: Publish dbgate
+
+      - name: Publish dbgate (obsolete)
         working-directory: packages/dbgate
         run: |
           npm publish
+
+      - name: Publish dbgate-serve
+        working-directory: packages/serve
+        run: |
+          npm publish
+
+      - name: Publish dbmodel
+        working-directory: packages/dbmodel
+        run: |
+          npm publish
+
+      - name: Publish dbgate-plugin-csv
+        working-directory: plugins/dbgate-plugin-csv
+        run: |
+          npm publish
+
+      - name: Publish dbgate-plugin-xml
+        working-directory: plugins/dbgate-plugin-xml
+        run: |
+          npm publish
+
+      - name: Publish dbgate-plugin-excel
+        working-directory: plugins/dbgate-plugin-excel
+        run: |
+          npm publish
+
+      - name: Publish dbgate-plugin-mssql
+        working-directory: plugins/dbgate-plugin-mssql
+        run: |
+          npm publish
+
+      - name: Publish dbgate-plugin-mysql
+        working-directory: plugins/dbgate-plugin-mysql
+        run: |
+          npm publish
+
+      - name: Publish dbgate-plugin-mongo
+        working-directory: plugins/dbgate-plugin-mongo
+        run: |
+          npm publish
+
+      - name: Publish dbgate-plugin-postgres
+        working-directory: plugins/dbgate-plugin-postgres
+        run: |
+          npm publish
+
+      - name: Publish dbgate-plugin-sqlite
+        working-directory: plugins/dbgate-plugin-sqlite
+        run: |
+          npm publish
+
+      - name: Publish dbgate-plugin-redis
+        working-directory: plugins/dbgate-plugin-redis
+        run: |
+          npm publish
+
+      - name: Publish dbgate-plugin-oracle
+        working-directory: plugins/dbgate-plugin-oracle
+        run: |
+          npm publish

.github/workflows/run-tests.yaml

@@ -0,0 +1,82 @@
+name: Run tests
+on:
+  push:
+    branches:
+      - master
+      - develop
+
+jobs:
+  test-runner:
+    runs-on: ubuntu-latest
+    container: node:14.18
+
+    steps:
+      - name: Context
+        env:
+          GITHUB_CONTEXT: ${{ toJson(github) }}
+        run: echo "$GITHUB_CONTEXT"
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 1
+      - name: yarn install
+        run: |
+          yarn install
+      - name: Integration tests
+        run: |
+          cd integration-tests
+          yarn test:ci
+#          yarn wait:ci
+      - name: Filter parser tests
+        if: always()
+        run: |
+          cd packages/filterparser
+          yarn test:ci
+      - name: Datalib (perspective) tests
+        if: always()
+        run: |
+          cd packages/datalib
+          yarn test:ci
+      - uses: tanmen/jest-reporter@v1
+        if: always()
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          result-file: integration-tests/result.json
+          action-name: Integration tests
+      - uses: tanmen/jest-reporter@v1
+        if: always()
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          result-file: packages/filterparser/result.json
+          action-name: Filter parser test results
+      - uses: tanmen/jest-reporter@v1
+        if: always()
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          result-file: packages/datalib/result.json
+          action-name: Datalib (perspectives) test results
+
+    services:
+      postgres:
+        image: postgres
+        env:
+          POSTGRES_PASSWORD: Pwd2020Db
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+
+      mysql:
+        image: mysql:8.0.18
+        env:
+          MYSQL_ROOT_PASSWORD: Pwd2020Db
+
+      mssql:
+        image: mcr.microsoft.com/mssql/server
+        env:
+          ACCEPT_EULA: Y
+          SA_PASSWORD: Pwd2020Db
+          MSSQL_PID: Express
+          
+      # cockroachdb:
+      #   image: cockroachdb/cockroach

.gitignore

@@ -12,6 +12,12 @@ node_modules
 build
 dist
 
+app/packages/web/public
+app/packages/plugins
+docker/public
+docker/bundle.js
+docker/plugins
+
 # misc
 .DS_Store
 .env.local
@@ -24,3 +30,5 @@ yarn-debug.log*
 yarn-error.log*
 app/src/nativeModulesContent.js
 packages/api/src/nativeModulesContent.js
+packages/api/src/packagedPluginsContent.js
+.VSCodeCounter

.node-version

@@ -0,0 +1 @@
+16.14.2

.vscode/restore-terminals.json

@@ -0,0 +1,20 @@
+{
+  "terminals": [
+    {
+      "splitTerminals": [
+        {
+          "name": "lib",
+          "commands": ["yarn lib"]
+        },
+        {
+          "name": "web",
+          "commands": ["yarn start:web"]
+        },
+        {
+          "name": "api",
+          "commands": ["yarn start:api"]
+        }
+      ]
+    }
+  ]
+}

.vscode/settings.json

@@ -0,0 +1,6 @@
+{
+    "jestrunner.jestCommand": "node_modules/.bin/cross-env DEVMODE=1 LOCALTEST=1 node_modules/.bin/jest",
+    "cSpell.words": [
+        "dbgate"
+    ]
+}

CHANGELOG.md

@@ -1,5 +1,628 @@
 # ChangeLog
 
+Builds:
+ - docker - build
+ - npm - npm package dbgate-serve
+ - app - classic electron app
+ - mac - application for macOS
+ - linux - application for linux
+ - win - application for Windows
+
+### 5.2.6
+- FIXED: DbGate creates a lot of .tmp.node files in the temp directory #561
+- FIXED: Typo in datetimeoffset dataType #556
+- FIXED: SQL export is using the wrong hour formatting #537
+- FIXED: Missing toolstrip and adds up to 200% zoom to diagram view #524
+- FIXED: MongoDB password could contain special characters #560
+
+### 5.2.5
+- ADDED: Split Windows #394
+- FIXED: Postgres index asc/desc #514
+- FIXED: Excel export not working since 5.2.3 #511
+- ADDED: Include macOS specific app icon #494
+- FIXED: Resizing window resets window contents #479
+- FIXED: Solved some minor problems with widget collapsing
+
+### 5.2.4
+- FIXED: npm version crash (#508)
+
+### 5.2.3
+- ADDED: Search entire table (multi column filter) #491
+- ADDED: OracleDB - connection to toher than default ports #496
+- CHANGED: OracleDB - status of support set to experimental
+- FIXED: OracleDB database URL - fixes: Connect to default Oracle database #489
+- ADDED: HTML, XML code highlighting for Edit cell value #485
+- FIXED: Intellisense - incorrect alias after ORDER BY clause #484
+- FIXED: Typo in SQL-Generator #481
+- ADDED: Data duplicator #480
+- FIXED: MongoDB - support for views #476
+- FIXED: "SQL:CREATE TABLE" generated SQL default value syntax errors #455
+- FIXED: Crash when right-clicking on tables #452
+- FIXED: View sort #436
+- ADDED: Arm64 version for Windows #473
+- ADDED: Sortable query results and data archive
+- CHANGED: Use transactions for saving table data
+- CHANGED: Save table structure uses transactions
+- ADDED: Table data editing - shows editing mark
+- ADDED: Editing data archive files
+- FIXED: Delete cascade options when using more than 2 tables
+- ADDED: Save to current archive commands
+- ADDED: Current archive mark is on status bar
+- FIXED: Changed package used for parsing JSONL files when browsing - fixes backend freezing
+- FIXED: SSL option for mongodb #504
+- REMOVED: Data sheet editor
+- FIXED: Creating SQLite autoincrement column
+- FIXED: Better error reporting from exports/import/dulicator
+- CHANGED: Optimalizede OracleDB analysing algorithm
+- ADDED: Mutli column filter for perspectives
+- FIXED: Fixed some scenarios using tables from different DBs
+- FIXED: Sessions with long-running queries are not killed
+
+
+### 5.2.2
+- FIXED: Optimalized load DB structure for PostgreSQL #451
+- ADDED: Auto-closing query connections after configurable (15 minutes default) no-activity interval #468
+- ADDED: Set application-name connection parameter (for PostgreSQL and MS SQL) for easier identifying of DbGate connections
+- ADDED: Filters supports binary IDs #467
+- FIXED: Ctrl+Tab works (switching tabs) #457
+- FIXED: Format code supports non-standard letters #450
+- ADDED: New logging system, log to file, ability to reduce logging #360 (using https://www.npmjs.com/package/pinomin)
+- FIXED: crash on Windows and Mac after system goes in suspend mode #458
+- ADDED: dbmodel standalone NPM package (https://www.npmjs.com/package/dbmodel) - deploy database via commandline tool
+
+
+### 5.2.1
+- FIXED: client_id param in OAuth
+- ADDED: OAuth scope parameter
+- FIXED: login page - password was not sent, when submitting by pressing ENTER
+- FIXED: Used permissions fix
+- FIXED: Export modal - fixed crash when selecting different database
+
+### 5.2.0
+- ADDED: Oracle database support #380
+- ADDED: OAuth authentification #407
+- ADDED: Active directory (Windows) authentification #261
+- ADDED: Ask database credentials when login to DB
+- ADDED: Login form instead of simple authorization (simple auth is possible with special configuration)
+- FIXED: MongoDB - connection uri regression
+- ADDED: MongoDB server summary tab
+- FIXED: Broken versioned tables in MariaDB #433
+- CHANGED: Improved editor margin #422
+- ADDED: Implemented camel case search in all search boxes
+- ADDED: MonhoDB filter empty array, not empty array
+- ADDED: Maximize button reflects window state
+- ADDED: MongoDB - database profiler
+- CHANGED: Short JSON values are shown directly in grid
+- FIXED: Fixed filtering nested fields in NDJSON viewer
+- CHANGED: Improved fuzzy search after Ctrl+P #246
+- ADDED: MongoDB: Create collection backup
+- ADDED: Single database mode
+- ADDED: Perspective designer supports joins from MongoDB nested documents and arrays
+- FIXED: Perspective designer joins on MongoDB ObjectId fields
+- ADDED: Filtering columns in designer (query designer, diagram designer, perspective designer)
+- FIXED: Clone MongoDB rows without _id attribute #404
+- CHANGED: Improved cell view with GPS latitude, longitude fields
+- ADDED: SQL: ALTER VIEW and SQL:ALTER PROCEDURE scripts
+- ADDED: Ctrl+F5 refreshes data grid also with database structure #428
+- ADDED: Perspective display modes: text, force text #439
+- FIXED: Fixed file filters #445
+- ADDED: Rename, remove connection folder, memoize opened state after app restart #425
+- FIXED: Show SQLServer alter store procedure #435
+
+
+### 5.1.6
+- ADDED: Connection folders support #274
+- ADDED: Keyboard shortcut to hide result window and show/hide the side toolbar #406
+- ADDED: Ability to show/hide query results #406
+- FIXED: Double click does not maximize window on MacOS #416
+- FIXED: Some perspective rendering errors
+- FIXED: Connection to MongoDB via database URL info SSH tunnel is used
+- CHANGED: Updated windows code signing certificate
+- ADDED: Query session cleanup (kill query sessions, if browser tab is closed)
+- CHANGED: More strict timeouts to kill database and server connections (reduces resource consumption)
+
+### 5.1.5
+- ADDED: Support perspectives for MongoDB - MongoDB query designer
+- ADDED: Show JSON content directly in the overview #395
+- CHANGED: OSX Command H shortcut for hiding window #390
+- ADDED: Uppercase Autocomplete Suggestions #389
+- FIXED: Record view left/right arrows cause start record number to be treated as string #388
+- FIXED: MongoDb ObjectId behaviour not consistent in nested objects #387
+- FIXED: demo.dbgate.org - beta version crash 5.1.5-beta.3 #386
+- ADDED: connect via socket - configurable via environment variables #358
+
+### 5.1.4
+- ADDED: Drop database commands #384
+- ADDED: Customizable Redis key separator #379
+- ADDED: ARM support for docker images
+- ADDED: Version tags for docker images
+- ADDED: Better SQL command splitting and highlighting
+- ADDED: Unsaved marker for SQL files
+
+### 5.1.3
+- ADDED: Editing multiline cell values #378 #371 #359
+- ADDED: Truncate table #333
+- ADDED: Perspectives - show row count
+- ADDED: Query - error markers in gutter area
+- ADDED: Query - ability to execute query elements from gutter
+- FIXED: Correct error line numbers returned from queries
+
+### 5.1.2
+- FIXED: MongoDb any export function does not work. #373
+- ADDED: Query Designer short order more flexibility #372
+- ADDED: Form View move between records #370
+- ADDED: Custom SQL conditions in query designer and table filtering #369
+- ADDED: Query Designer filter eq to X or IS NULL #368
+- FIXED: Query designer, open a saved query lost sort order #363
+- ADDED: Query designer reorder columns #362
+- ADDED: connect via socket #358
+- FIXED: Show affected rows after UPDATE/DELETE/INSERT #361
+- ADDED: Perspective cell formatters - JSON, image
+- ADDED: Perspectives - cells without joined data are gray
+
+### 5.1.1
+- ADDED: Perspective designer
+- FIXED: NULL,NOT NULL filter datatime columns #356
+- FIXED: Recognize computed columns on SQL server #354
+- ADDED: Hotkey for clear filter #352
+- FIXED: Change column type on Postgres #350
+- ADDED: Ability to open qdesign file #349
+- ADDED: Custom editor font size #345
+- ADDED: Ability to open perspective files
+
+
+### 5.1.0
+- ADDED: Perspectives (docs: https://dbgate.org/docs/perspectives.html )
+- CHANGED: Upgraded SQLite engine version (driver better-sqlite3: 7.6.2)
+- CHANGED: Upgraded ElectronJS version (from version 13 to version 17)
+- CHANGED: Upgraded all dependencies with current available minor version updates
+- CHANGED: By default, connect on click #332˝
+- CHANGED: Improved keyboard navigation, when editing table data #331
+- ADDED:  Option to skip Save changes dialog #329
+- FIXED: Unsigned column doesn't work correctly. #324
+- FIXED: Connect to MS SQL with domain user now works also under Linux and Mac #305
+
+### 5.0.9
+- FIXED: Fixed problem with SSE events on web version
+- ADDED: Added menu command "New query designer"
+- ADDED: Added menu command "New ER diagram"
+
+### 5.0.8
+- ADDED: SQL Server - support using domain logins under Linux and Mac #305
+- ADDED: Permissions for connections #318
+- ADDED: Ability to change editor front #308
+- ADDED: Custom expression in query designer #306
+- ADDED: OR conditions in query designer #321
+- ADDED: Ability to configure settings view environment variables #304
+ 
+### 5.0.7
+- FIXED: Fixed some problems with SSH tunnel (upgraded SSH client) #315
+- FIXED: Fixed MognoDB executing find query #312
+- ADDED: Interval filters for date/time columns #311
+- ADDED: Ability to clone rows #309
+- ADDED: connecting option Trust server certificate for SQL Server #305
+- ADDED: Autorefresh, reload table every x second #303
+- FIXED(app): Changing editor theme and font size in Editor Themes #300
+ 
+### 5.0.6
+- ADDED: Search in columns
+- CHANGED: Upgraded mongodb driver
+- ADDED: Ability to reset view, when data load fails
+- FIXED: Filtering works for complex types (geography, xml under MSSQL)
+- FIXED: Fixed some NPM package problems
+
+### 5.0.5
+- ADDED: Visualisation geographics objects on map #288
+- ADDED: Support for native SQL as default value inside yaml files #296
+- FIXED: Postgres boolean columns don't filter correctly #298
+- FIXED: Importing dbgate-api as NPM package now works correctly
+- FIXED: Handle error when reading deleted archive
+
+### 5.0.3
+- CHANGED: Optimalization of loading DB structure for PostgreSQL, MySQL #273
+- CHANGED: Upgraded mysql driver #293
+- CHANGED: Better UX when defining SSH port #291
+- ADDED: Database object menu from tab 
+- CHANGED: Ability to close file uploader
+- FIXED: Correct handling of NUL values in update keys
+- CHANGED: Upgraded MS SQL tedious driver
+- ADDED: Change order of pinned tables & databases #227
+- FIXED: #294 Statusbar doesn't match active tab
+- CHANGED: Improved connection worklflow, disconnecting shws confirmations, when it leads to close any tabs
+- ADDED: Configurable object actions #255
+- ADDED: Multiple sort criteria #235
+- ADDED(app): Open JSON file
+### 5.0.2
+- FIXED: Cannot use SSH Tunnel after update #291
+
+### 5.0.1
+- FIXED(app): Can't Click Sidebar Menu Item #287
+
+### 5.0.0
+- CHANGED: Connection workflow, connections are opened on tabs instead of modals
+- ADDED: Posibility to connect to DB without saving connection
+- ADDED(mac): Support for SQLite on Mac M1
+- FIXED(mac): Unable to drag window on MacOS #281 #283
+- CHANGED: Renamed dbgate-data directory to .dbgate #248
+- FIXED: Exported SQL has table name undefined #277
+- ADDED: More data types in table create dialogt #285
+- ADDED(app): Open previously saved ERD diagrams #278
+- CHANGED: Better app loading progress UX
+- FIXED: Removed SSL tab on Redis connection (SSL is not supported for Redis)
+
+### 4.8.8
+- CHANGED: New app icon
+- ADDED: SQL dump, SQL import - also from/to saved queries
+- FIXED(mac): Fixed crash when reopening main window
+- FIXED: MySQL dump now handles correctly dependand views
+- FIXED(app): Browse tabs with Ctrl+Tab
+- ADDED(app): Browse tabs in reverse order with Ctrl+Shift+Tab #245
+
+### 4.8.7
+- ADDED: MySQL dump/backup database
+- ADDED: Import SQL dump from file or from URL
+- FIXED(mac): Fixed Cmd+C, Cmd+V, Cmd+X - shortcuts for copy/cut/paste #270
+- FIXED(mac): Some minor issues on macOS
+- FIXED: Analysing MS SQL nvarchar(max)
+- ADDED: Support for dockerhost network name under docker #271
+
+### 4.8.4
+- FIXED(mac): Fixed build for macOS arm64 #259
+- FIXED(mac): Fixed opening SQLite files on macOS #243
+- FIXED(mac): Fixed opening PEM certificates on macOS #206
+- FIXED(mac): Fixed handling Command key on macOS
+- FIXED(mac): Fixed system menu on macOS
+- FIXED(mac): Fixed reopening main window on macOS
+- CHANGED: Shortcut for net query is now Ctrl+T or Command+T on macOS, former it was Ctrl+Q
+- FIXED: Fixed misplaced tab close icon #260
+- ADDED: Added menu command "Tools/Change to recent database"
+
+### 4.8.3
+- FIXED: filters in query result and NDJSON/archive viewer
+- ADDED: Added select values from query result and NDJSON/archive viewer
+- ADDED: tab navigation in datagrid #254
+- ADDED: Keyboard shortcuts added to help menu #254
+- ADDED: API logging (run enableApiLog() in developers console to enable logging)
+- ADDED: SSH reconnect + moved SSH forward into separate fork #253
+- ADDED: Data type + reference link in column manager
+- FIXED(win,linux,mac): Unable to change theme after installing plugin #244
+
+ ### 4.8.2
+ - ADDED: implemented missing redis search key logic
+
+ ### 4.8.1
+ - FIXED: fixed crash after disconnecting from all DBs
+
+### 4.8.0
+- ADDED: Redis support (support stream type), removed experimental status
+- ADDED: Redis readonly support
+- ADDED: Explicit NDJSON support, when opening NDJSON/JSON lines file, table data are immediately shown, without neccesarity to import
+- ADDED(win,linux,mac): Opening developer tools when crashing without reload app
+### 4.7.4
+- ADDED: Experimental Redis support (full support is planned to version  4.8.0)
+- ADDED: Read-only connections
+- FIXED: MongoDB filters
+- ADDED: MongoDB column value selection
+- ADDED: App related queries
+- ADDED: Fuzzy search #246
+- ADDED(docker, npm): New permissions
+- FIXED(npm): NPM build no longer allocates additonal ports
+- CHANGED(npm): renamed NPM package dbgate => dbgate-serve 
+- CHANGED(docker): custom JavaScripts and connections defined in scripts are now prohibited by default, use SHELL_CONNECTION and SHELL_SCRIPTING environment variables for allowing this
+- ADDED(docker, npm): Better documentation of environment variables configuration, https://dbgate.org/docs/env-variables.html
+- ADDED(docker): support for multiple users with different permissions
+- ADDED(docker): logout operation
+
+### 4.7.3
+- CHANGED: Export menu redesign, quick export menu merged with old export menu
+- REMOVED: Quick export menu
+- ADDED: Export column mapping
+- ADDED: Export invoked from data grid respects columns choosed in column manager
+- ADDED: Quick export (now merged in export menu) is now possible also in web app
+- FIXED: Virtual foreign key editor fixes
+- FIXED: Tabs panel style fix
+- ADDED: Find by schema in databases widget
+- FIXED: Column manager selection fix
+- FIXED: NPM dist - fixed error when loading plugins
+- CHANGED: NPN dist is now executed by dbgate-serve command
+- ADDED: NPM dist accepts .env configuration
+
+### 4.7.2
+- CHANGED: documentation URL - https://dbgate.org/docs/
+- CHANGED: Close button available for all tab groups - #238
+- ADDED: Search function for the Keyboard Shortcuts overview - #239
+- ADDED: Editor font size settings - #229
+- ADDED: Rename MongoDB collection - #223
+- FIXED: bug in cache subsystem
+
+### 4.7.1
+- FIXED: Fixed connecting to MS SQL server running in docker container from DbGate running in docker container #236 
+- FIXED: Fixed export MongoDB collections into Excel and CSV #240
+- ADDED: Added support for docker volumes to persiste connections, when not using configuration via env variables #232
+- ADDED: DbGate in Docker can run in subdirectory #228
+- FIXED: DbGate in Docker can be proxied with nginx #228
+- FIDED: Theme persists when opening multiple windows #207
+- ADDED: Remember fullscreen state #230
+- ADDED: Improved fullscreen state, title bar with menu is hidden, menu is in hamburger menu, like in web version
+- ADDED: Theme choose dialog (added as tab in settings)
+- FIXED: Fixed crash when clicking on application layers #231
+### 4.7.0
+- CHANGED: Changed main menu style, menu and title bar is in one line (+ability to switch to system menu)
+- REMOVED: Removed main toolbar, use main menu or tab related bottom tool instead
+- ADDED: Added tab related context bottom toolbar
+- ADDED: Main menu is available also in web application, by clicking on hamburger menu
+- ADDED: Added support of SQLite to docker container #219
+- ADDED: Added Debian and Alpine docker distributions (default is Debian)
+- FIXED: Fixed performance problem of data grid, especially when there are cells with large data (eg. JSONs), now it is much faster
+- ADDED: Open JSON and array cell buttons
+- ADDED: Handle JSON in varchar cells
+- ADDED: Scroll tabs on mouse wheel
+- ADDED: Show edit edit MySQL column comments #218 #81
+- ADDED: Handle sparse (mssql), unsigned (mysql), zerofill (mysql) column flags
+- FIXED: Fixed same caching problems (eg. leading to indefinitely loading DB structure sometimes)
+- ADDED: Show estimated table row count for MySQL and MS SQL
+- FIXED: Fixed deleting rows from added rows in table data editor
+- ADDED: Better work with JSON lines file, added JSONL editor with preview
+
+### 4.6.3
+- FIXED: Fixed Windows build
+- FIXED: Fixed crash, when there is invalid value in browser local storage
+- FIXED: Fixed plugin description display, where author name or description is not correctly filled
+
+### 4.6.2
+- FIXED: Fixed issues of XML import plugin
+- ADDED: Split columns macro (available in data sheet editor)
+- CHANGED: Accepting non standard plugins names (which doesn't start with dbgate-plugin-)
+- ADDED: Support BLOB values #211
+- ADDED: Picture cell view
+- ADDED: HTML cell view
+- CHANGED: Code completion supports non-default schema names
+- FIXED: More robust MySQL analyser, when connecting to non-standard servers #214
+- FIXED: Fixed configuring connection to SQLite with environment variables #215
+
+### 4.6.1
+- ADDED: Ability to configure SSH tunnel over environment variables #210 (for docker container)
+- ADDED: XML export and import
+- ADDED: Archive file - show and edit source text file
+- ADDED: Window title shows current tab and database
+- ADDED: DbGate documentation
+- ADDED: Introduced application layers
+- ADDED: Virtual foreign key editor
+- ADDED: Application commands (SQL scripts related to database)
+- ADDED: Theme can be implemented in plugin
+- CHANGED: Dictionary description is stored in app
+- FIXED: Unique and index editor
+- FIXED: Posibility to edit UNIQUE index flag
+- CHANGED: UX improvements of table editor
+
+### 4.6.0
+- ADDED: ER diagrams #118
+    - Generate diagram from table or for database
+    - Automatic layout
+    - Diagram styles - colors, select columns to display, optional displaying data type or nullability
+    - Export diagram to HTML file
+- FIXED: Mac latest build link #204
+
+### 4.5.1
+- FIXED: MongoId detection
+- FIXED: #203 disabled spellchecker
+- FIXED: Prevented display filters in form view twice
+- FIXED: Query designer fixes
+
+### 4.5.0
+- ADDED: #220 functions, materialized views and stored procedures in code completion
+- ADDED: Query result in statusbar
+- ADDED: Highlight and execute current query
+- CHANGED: Code completion offers objects only from current query
+- CHANGED: Big optimalizations of electron app - removed embedded web server, removed remote module, updated electron to version 13
+- CHANGED: Removed dependency to electron-store module
+- FIXED: #201 fixed database URL definition, when running from Docvker container
+- FIXED: #192 Docker container stops in 1 second, ability to stop container with Ctrl+C
+- CHANGED: Web app - websocket replaced with SSE technology
+- CHANGED: Changed tab order, tabs are ordered by creation time
+- ADDED: Reorder tabs with drag & drop
+- CHANGED: Collapse left column in datagrid - removed from settings, remember last used state
+- ADDED: Ability to select multiple columns in column manager in datagrid + copy column names
+- ADDED: Show used filters in left datagrid column
+- FIXED: Fixed delete dependency cycle detection (delete didn't work for some tables)
+
+### 4.4.4
+- FIXED: Database colors
+- CHANGED: Precise work with MongoDB ObjectId
+- FIXED: Run macro works on MongoDB collection data editor
+- ADDED: Type conversion macros
+- CHANGED: Improved UX of import into current database or current archive
+- ADDED: Posibility to create string MongoDB IDs when importing into MongoDB collections
+- CHANGED: Better crash recovery
+- FIXED: Context menu of data editor when using views - some commands didn't work for views
+- ADDED: Widget lists (on left side) now supports add operation, where it has sense
+- CHANGED: Improved UX of saved data sheets
+- ADDED: deploy - preloadedRows: impelemnted onsertOnly columns
+- ADDED: Show change log after app upgrade
+
+### 4.4.3
+- ADDED: Connection and database colors
+- ADDED: Ability to pin connection or table
+- ADDED: MongoDb: create, drop collection from menu
+- ADDED: Copy as MongoDB insert
+- ADDED: MongoDB support for multiple statements in script (dbgate-query-splitter)
+- ADDED: View JSON in tab
+- ADDED: Open DB model as JSON
+- ADDED: Open JSON array as data sheet
+- ADDED: Open JSON from data grid
+- FIXED: Mongo update command when using string IDs resembling Mongo IDs
+- CHANGED: Imrpoved add JSON document, change JSON document commands
+- ADDED: Possibility to add column to JSON grid view
+- FIXED: Hiding columns #1
+- REMOVED: Copy JSON document menu command (please use Copy advanced instead)
+- CHANGED: Save widget visibility and size
+
+### 4.4.2
+- ADDED: Open SQL script from SQL confirm
+- CHANGED: Better looking statusbar
+- ADDED: Create table from database popup menu
+- FIXED: Some fixes for DB compare+deploy (eg. #196)
+- ADDED: Archives + DB models from external directories
+- ADDED: DB deploy supports preloaded data
+- ADDED: Support for Command key on Mac (#199)
+
+### 4.4.1
+- FIXED: #188 Fixed problem with datetime values in PostgreSQL and mysql
+- ADDED: #194 Close tabs by DB
+- FIXED: Improved form view width calculations
+- CHANGED: Form view - highlight matched columns instead of filtering
+- ADDED: Lookup distinct values
+- ADDED: Copy advanced command, Copy as CSV, JSON, YAML, SQL
+- CHANGED: Hide column manager by default
+- ADDED: Change database status command
+- CHANGED: Table structure and view structure tabs have different icons
+- ADDED: #186 - zoom setting
+- ADDED: Row count information moved into status bar, when only one grid on tab is used (typical case)
+
+### 4.4.0
+- ADDED: Database structure compare, export report to HTML
+- ADDED: Experimental: Deploy DB structure changes between databases
+- ADDED: Lookup dialog, available in table view on columns with foreign key
+- ADDED: Customize foreign key lookups
+- ADDED: Chart improvements, export charts as HTML page
+- ADDED: Experimental: work with DB model, deploy model, compare model with real DB
+- ADDED: #193 new SQLite db command
+- CHANGED: #190 code completion improvements
+- ADDED: #189 Copy JSON document - context menu command in data grid for MongoDB
+- ADDED: #191 Connection to POstgreSQL can be defined also with connection string
+- ADDED: #187 dbgate-query-splitter: Transform stream support
+- CHANGED: Upgraded to node 12 in docker app
+- FIXED: Upgraded to node 12 in docker app
+- FIXED: Fixed import into SQLite and PostgreSQL databases, added integration test for this
+
+### 4.3.4
+- FIXED: Delete row with binary ID in MySQL (#182)
+- ADDED: Using 'ODBC Driver 17 for SQL Server' or 'SQL Server Native Client 11.0', when connecting to MS SQL using windows auth #183
+
+### 4.3.3
+- ADDED: Generate SQL from data (#176 - Copy row as INSERT/UPDATE statement)
+- ADDED: Datagrid keyboard column operations (Ctrl+F - find column, Ctrl+H - hide column) #180
+- FIXED: Make window remember that it was maximized
+- FIXED: Fixed lost focus after copy to clipboard and after inserting SQL join
+
+### 4.3.2
+- FIXED: Sorted database list in PostgreSQL (#178)
+- FIXED: Loading stricture of PostgreSQL database, when it contains indexes on expressions (#175)
+- ADDED: Hotkey Shift+Alt+F for formatting SQL code
+
+### 4.3.1
+- FIXED: #173 Using key phrase for SSH key file connection
+- ADDED: #172 Abiloity to quick search within database names
+- ADDED: Database search added to command palette (Ctrl+P)
+- FIXED: #171 fixed PostgreSQL analyser for older versions than 9.3 (matviews don't exist)
+- ADDED: DELETE cascade option - ability to delete all referenced rows, when deleting rows
+
+### 4.3.0
+- ADDED: Table structure editor
+- ADDED: Index support
+- ADDED: Unique constraint support
+- ADDED: Context menu for drop/rename table/columns and for drop view/procedure/function
+- ADDED: Added support for Windows arm64 platform
+- FIXED: Search by _id in MongoDB
+
+### 4.2.6
+- FIXED: Fixed MongoDB import
+- ADDED: Configurable thousands separator #136
+- ADDED: Using case insensitive text search in postgres
+
+### 4.2.5
+- FIXED: Fixed crash when using large model on some installations
+- FIXED: Postgre SQL CREATE function 
+- FIXED: Analysing of MySQL when modifyDate is not known
+
+### 4.2.4
+- ADDED: Query history
+- ADDED: One-click exports in desktop app
+- ADDED: JSON array export
+- FIXED: Procedures in PostgreSQL #122
+- ADDED: Support of materialized views for PostgreSQL #123
+- ADDED: Integration tests
+- FIXED: Fixes in DB structure analysis in PostgreSQL, SQLite, MySQL
+- FIXED: Save data in SQLite, PostgreSQL
+- CHANGED: Introduced package dbgate-query-splitter, instead of sql-query-identifier and @verycrazydog/mysql-parse
+
+### 4.2.3
+- ADDED: ARM builds for MacOS and Linux
+- ADDED: Filter by columns in form view
+
+### 4.2.2
+- CHANGED: Further startup optimalization (approx. 2 times quicker start of electron app)
+
+### 4.2.1
+- FIXED: Fixed+optimalized app startup (esp. on Windows)
+
+### 4.2.0
+- ADDED: Support of SQLite database
+- ADDED: Support of Amazon Redshift database
+- ADDED: Support of CockcroachDB
+- CHANGED: DB Model is not auto-refreshed by default, refresh could be invoked from statusbar
+- FIXED: Fixed race conditions on startup
+- FIXED: Fixed broken style in data grid under strange circumstances
+- ADDED: Configure connections with commandline arguments  #108
+- CHANGED: Optimalized algorithm of incremental DB model updates
+- CHANGED: Loading queries from PostgreSQL doesn't need cursors, using streamed query instead
+- ADDED: Disconnect command
+- ADDED: Query executed on server has tab marker (formerly it had only "No DB" marker)
+- ADDED: Horizontal scroll using shift+mouse wheel #113
+- ADDED: Cosmetic improvements of MariaDB support
+
+### 4.1.11
+- FIX: Fixed crash of API process when using SSH tunnel connection (race condition)
+
+### 4.1.11
+- FIX: fixed processing postgre query containing $$
+- FIX: fixed postgre analysing procedures & functions
+- FIX: patched svelte crash #105
+- ADDED: ability to disbale background DB model updates
+- ADDED: Duplicate connection
+- ADDED: Duplicate tab
+- FIX: SSH tunnel connection using keyfile auth #106
+- FIX: All tables button fix in export #109
+- CHANGED: Add to favorites moved from toolbar to tab context menu
+- CHANGED: Toolbar design - current tab related commands are delimited
+
+### 4.1.10
+- ADDED: Default database option in connectin settings #96 #92
+- FIX: Bundle size optimalization for Windows #97
+- FIX: Popup menu placement on smaller displays #94
+- ADDED: Browse table data with SQL Server 2008 #93
+- FIX: Prevented malicious origins / DNS rebinding #91
+- ADDED: Handle JSON fields in data editor (eg. jsonb field in Postgres) #90
+- FIX: Fixed crash on Windows with Hyper-V #86
+- ADDED: Show database server version in status bar
+- ADDED: Show detailed info about error, when connect to database fails
+- ADDED: Portable ZIP distribution for Windows #84
+### 4.1.9
+- FIX: Incorrect row count info in query result #83
+
+### 4.1.1
+- CHANGED: Default plugins are now part of installation
+### 4.1.0
+- ADDED: MongoDB support
+- ADDED: Configurable keyboard shortcuts
+- ADDED: JSON row cell data view
+- FIX: Fixed some problems from migration to Svelte
+
+### 4.0.3
+- FIX: fixes for FireFox (mainly incorrent handle of bind:clientHeight, replaces with resizeobserver)
+### 4.0.2
+- FIX: fixed docker and NPM build
+### 4.0.0
+- CHANGED: Excahnged React with Svelte. Changed theme colors. Huge speed and memory optimalization
+- ADDED: SQL Generator (CREATE, INSERT, DROP)
+- ADDED: Command palette (F1). Introduced commands, extended some context menus
+- ADDED: New keyboard shortcuts
+- ADDED: Switch to recent database feature
+- ADDED: Macros from free table editor are available also in table data editor
+- CHANGED: Cell data preview is now in left widgets panel
+- CHANGED: Toolbar refactor
+- FIX: Solved reconnecting expired connection
+
 ### 3.9.6
 - ADDED: Connect using SSH Tunnel
 - ADDED: Connect using SSL

README.md

@@ -1,53 +1,108 @@
-[![NPM version](https://img.shields.io/npm/v/dbgate.svg)](https://www.npmjs.com/package/dbgate)
+[![NPM version](https://img.shields.io/npm/v/dbgate-serve.svg)](https://www.npmjs.com/package/dbgate-serve)
+![GitHub All Releases](https://img.shields.io/github/downloads/dbgate/dbgate/total) 
 [![dbgate](https://snapcraft.io/dbgate/badge.svg)](https://snapcraft.io/dbgate)
 [![dbgate](https://snapcraft.io/dbgate/trending.svg?name=0)](https://snapcraft.io/dbgate)
 [![styled with prettier](https://img.shields.io/badge/styled_with-prettier-ff69b4.svg)](https://github.com/prettier/prettier)
-[![Donate](https://img.shields.io/badge/donate-paypal-blue.svg)](https://paypal.me/JanProchazkaCz/30eur)
 
-# DbGate - database administration tool
+<img src="https://raw.githubusercontent.com/dbgate/dbgate/master/app/icon.png" width="64" align="right"/>
 
-DbGate is fast and easy to use database manager. Works with MySQL, PostgreSQL and SQL Server.
+# DbGate - (no)SQL database client
+
+DbGate is cross-platform database manager. 
+It's designed to be simple to use and effective, when working with more databases simultaneously.
+But there are also many advanced features like schema compare, visual query designer, chart visualisation or batch export and import.
+
+DbGate is licensed under MIT license and is completely free.
 
 * Try it online - [demo.dbgate.org](https://demo.dbgate.org) - online demo application
-* Download application for Windows, Linux or Mac from [dbgate.org](https://dbgate.org/download/)
-* Run web version as [NPM package](https://www.npmjs.com/package/dbgate) or as [docker image](https://hub.docker.com/r/dbgate/dbgate)
+* **Download** application for Windows, Linux or Mac from [dbgate.org](https://dbgate.org/download/)
+* Run web version as [NPM package](https://www.npmjs.com/package/dbgate-serve) or as [docker image](https://hub.docker.com/r/dbgate/dbgate)
 
-![Screenshot](https://raw.githubusercontent.com/dbgate/dbgate/master/screenshot.png)
+## Supported databases
+* MySQL
+* PostgreSQL
+* SQL Server
+* Oracle (experimental)
+* MongoDB
+* Redis
+* SQLite
+* Amazon Redshift
+* CockroachDB
+* MariaDB
+
+<!-- Learn more about DbGate features at the [DbGate website](https://dbgate.org/), or try our online [demo application](https://demo.dbgate.org) -->
+
+
+<a href="https://raw.githubusercontent.com/dbgate/dbgate/master/img/screenshot1.png">
+    <img src="https://raw.githubusercontent.com/dbgate/dbgate/master/img/screenshot1.png" width="400"/>
+</a>
+<a href="https://raw.githubusercontent.com/dbgate/dbgate/master/img/screenshot2.png">
+    <img src="https://raw.githubusercontent.com/dbgate/dbgate/master/img/screenshot2.png" width="400"/>
+</a>
+<a href="https://raw.githubusercontent.com/dbgate/dbgate/master/img/screenshot4.png">
+    <img src="https://raw.githubusercontent.com/dbgate/dbgate/master/img/screenshot4.png" width="400"/>
+</a>
+<a href="https://raw.githubusercontent.com/dbgate/dbgate/master/img/screenshot3.png">
+    <img src="https://raw.githubusercontent.com/dbgate/dbgate/master/img/screenshot3.png" width="400"/>
+</a>
+
+<!-- ![Screenshot](https://raw.githubusercontent.com/dbgate/dbgate/master/screenshot.png) -->
 
 ## Features
-* Connect to Microsoft SQL Server, Postgre SQL, MySQL
 * Table data editing, with SQL change script preview
-* Master/detail views
+* Edit table schema, indexes, primary and foreign keys
+* Compare and synchronize database structure
+* ER diagram
+* Light and dark theme
+* Master/detail views, foreign key lookups
 * Query designer
 * Form view for comfortable work with tables with many columns
-* Explore tables, views, procedures, functions
-* SQL editor, execute SQL script, SQL code formatter, SQL code completion, SQL join wizard
+* JSON view on MongoDB collections
+* Explore tables, views, procedures, functions, MongoDB collections
+* SQL editor
+  * execute SQL script
+  * SQL code formatter
+  * SQL code completion
+  * Add SQL LEFT/INNER/RIGHT join utility
+* Mongo JavaScript editor, execute Mongo script (with NodeJs syntax)
+* Redis tree view, generate script from keys, run Redis script
 * Runs as application for Windows, Linux and Mac. Or in Docker container on server and in web Browser on client.
-* Import, export from/to CSV, Excel, JSON
+* Import, export from/to CSV, Excel, JSON, NDJSON, XML
 * Free table editor - quick table data editing (cleanup data after import/before export, prototype tables etc.)
-* Archives - backup your data in JSON files on local filesystem (or on DbGate server, when using web application)
-* Light and dark theme
-* Charts
+* Archives - backup your data in NDJSON files on local filesystem (or on DbGate server, when using web application)
+* Charts, export chart to HTML page
 * For detailed info, how to run DbGate in docker container, visit [docker hub](https://hub.docker.com/r/dbgate/dbgate)
 * Extensible plugin architecture
+* Perspectives - nested table view over complex relational data, query designer on MongoDB databases
+
+## How to contribute
+Any contributions are welcome. If you want to contribute without coding, consider following:
+
+* Tell your friends about DbGate or share on social networks - when more people will use DbGate, it will grow to be better
+* Write review on [Slant.co](https://www.slant.co/improve/options/41086/~dbgate-review) or [G2](https://www.g2.com/products/dbgate/reviews) 
+* Create issue, if you find problem in app, or you have idea to new feature. If issue already exists, you could leave comment on it, to prioritise most wanted issues
+* Create some tutorial video on [youtube](https://www.youtube.com/playlist?list=PLCo7KjCVXhr0RfUSjM9wJMsp_ShL1q61A)
+* Become a backer on [GitHub sponsors](https://github.com/sponsors/dbgate) or [Open collective](https://opencollective.com/dbgate)
+* Where a small coding is acceptable for you, you could [create plugin](https://dbgate.org/docs/plugin-development.html). Plugins for new themes can be created actually without JS coding
+
+Thank you!
 
 ## Why is DbGate different
 There are many database managers now, so why DbGate? 
 * Works everywhere - Windows, Linux, Mac, Web browser (+mobile web is planned), without compromises in features
 * Based on standalone NPM packages, scripts can be run without DbGate (example - [CSV export](https://www.npmjs.com/package/dbgate-plugin-csv) )
-* Many data browsing functions based using foreign keys - master/detail, expand columns, expandable form view (on screenshot above)
+* Many data browsing functions based using foreign keys - master/detail, expand columns, expandable form view
 
 ## Design goals
-* Application simplicity - DbGate takes the best and only the best from old [DbGate](http://www.jenasoft.com/dbgate), [DatAdmin](http://www.jenasoft.com/datadmin) and [DbMouse](http://www.jenasoft.com/dbmouse) .
+* Application simplicity - DbGate takes the best and only the best from old DbGate, [DatAdmin](https://www.softpedia.com/get/Internet/Servers/Database-Utils/DatAdmin-Personal.shtml), [DbMouse](https://www.softpedia.com/get/Internet/Servers/Database-Utils/DbMouse.shtml) and [SQL Database Studio](https://en.wikipedia.org/wiki/SQL_Database_Studio)
 * Minimal dependencies
-    * Frontend - React, styled-components, socket.io
-    * Backend - NodeJs, ExpressJs, socket.io, database connection drivers
+    * Frontend - Svelte
+    * Backend - NodeJs, ExpressJs, database connection drivers
     * JavaScript + TypeScript
     * App - electron
-    * There is plan to incorporate SQLite to support work with local datasets
-* Platform independed - will run as web application in single docker container on server, or as application using Electron platform on Linux, Windows and Mac
+* Platform independent - runs as web application in single docker container on server, or as application using Electron platform on Linux, Windows and Mac
 
-## Plugins
+<!-- ## Plugins
 Plugins are standard NPM packages published on [npmjs.com](https://www.npmjs.com).  
 See all [existing DbGate plugins](https://www.npmjs.com/search?q=keywords:dbgateplugin).  
 Visit [dbgate generator homepage](https://github.com/dbgate/generator-dbgate) to see, how to create your own plugin.  
@@ -56,29 +111,46 @@ Currently following extensions can be implemented using plugins:
 - File format parsers/writers
 - Database engine connectors
 
+Basic set of plugins is part of DbGate git repository and is installed with app. Additional plugins pust be downloaded from NPM (this task is handled by DbGate) -->
+
 ## How to run development environment
 
+Simple variant - runs WEB application:
 ```sh
 yarn
 yarn start
 ```
 
-If you want to make modifications in TypeScript packages, run TypeScript compiler in watch mode in seconds terminal:
+If you want more control, run WEB application:
 ```sh
-yarn lib
+yarn # install NPM packages
 ```
 
-Open http://localhost:5000 in your browser
+And than run following 3 commands concurrently in 3 terminals:
+```
+yarn start:api # run API on port 3000
+yarn start:web # run web on port 5001
+yarn lib # watch typescript libraries and plugins modifications
+```
+This runs API on port 3000 and web application on port 5001  
+Open http://localhost:5001 in your browser
 
-You could run electron app (requires running localhost:5000):
+If you want to run electron app:
 ```sh
+yarn # install NPM packages
 cd app
-yarn
-yarn start
+yarn # install NPM packages for electron
+```
+
+And than run following 3 commands concurrently in 3 terminals:
+```
+yarn start:web # run web on port 5001 (only static JS and HTML files)
+yarn lib # watch typescript libraries and plugins modifications
+yarn start:app # run electron app
 ```
 
 ## How to run built electron app locally
-This mode is very similar to production run of electron app. Electron app forks process with API on dynamically allocated port, works with compiled javascript files (doesn't use localhost:5000)
+This mode is very similar to production run of electron app. Electron doesn't use localhost:5001.
 
 ```sh
 cd app
@@ -91,15 +163,19 @@ yarn build:app:local
 yarn start:app:local
 ```
 
-## Packages
-Some dbgate packages can be used also without DbGate. You can find them on [NPM repository](https://www.npmjs.com/search?q=keywords:dbgate)
+## How to create plugin
+Creating plugin is described in [documentation](https://github.com/dbgate/dbgate/wiki/Plugin-development)
 
-* [api](https://github.com/dbgate/dbgate/tree/master/packages/api) - backend, Javascript, ExpressJS [![NPM version](https://img.shields.io/npm/v/dbgate-api.svg)](https://www.npmjs.com/package/dbgate-api)
-* [datalib](https://github.com/dbgate/dbgate/tree/master/packages/datalib) - TypeScript library for utility classes [![NPM version](https://img.shields.io/npm/v/dbgate-datalib.svg)](https://www.npmjs.com/package/dbgate-datalib)
-* [app](https://github.com/dbgate/dbgate/tree/master/app) - application (JavaScript) structure, creating specific queries (JavaScript)
-* [filterparser](https://github.com/dbgate/dbgate/tree/master/packages/filterparser) - TypeScript library for parsing data filter expressions using parsimmon [![NPM version](https://img.shields.io/npm/v/dbgate-filterparser.svg)](https://www.npmjs.com/package/dbgate-filterparser)
-* [sqltree](https://github.com/dbgate/dbgate/tree/master/packages/sqltree) - JSON representation of SQL query, functions converting to SQL (TypeScript) [![NPM version](https://img.shields.io/npm/v/dbgate-sqltree.svg)](https://www.npmjs.com/package/dbgate-sqltree)
-* [types](https://github.com/dbgate/dbgate/tree/master/packages/types) - common TypeScript definitions [![NPM version](https://img.shields.io/npm/v/dbgate-types.svg)](https://www.npmjs.com/package/dbgate-types)
-* [web](https://github.com/dbgate/dbgate/tree/master/packages/web) - frontend in React (JavaScript) [![NPM version](https://img.shields.io/npm/v/dbgate-web.svg)](https://www.npmjs.com/package/dbgate-web)
-* [tools](https://github.com/dbgate/dbgate/tree/master/packages/tools) - various tools [![NPM version](https://img.shields.io/npm/v/dbgate-tools.svg)](https://www.npmjs.com/package/dbgate-tools)
+But it is very simple:
 
+```sh
+npm install -g yo # install yeoman
+npm install -g generator-dbgate # install dbgate generator
+cd dbgate-plugin-my-new-plugin # this directory is created by wizard, edit, what you need to change
+yarn plugin # this compiles plugin and copies it into existing DbGate installation
+```
+
+After restarting DbGate, you could use your new plugin from DbGate.
+
+## Logging
+DbGate uses [pinomin logger](https://github.com/dbgate/pinomin). So by default, it produces JSON log messages into console and log files. If you want to see formatted logs, please use [pino-pretty](https://github.com/pinojs/pino-pretty) log formatter.

adjustPackageJson.js

@@ -0,0 +1,15 @@
+const fs = require('fs');
+
+function adjustFile(file) {
+  const json = JSON.parse(fs.readFileSync(file, { encoding: 'utf-8' }));
+  if (process.platform != 'win32') {
+    delete json.optionalDependencies.msnodesqlv8;
+  }
+  if (process.arch == 'arm64') {
+    delete json.optionalDependencies.oracledb;
+  }
+  fs.writeFileSync(file, JSON.stringify(json, null, 2), 'utf-8');
+}
+
+adjustFile('packages/api/package.json');
+adjustFile('app/package.json');

app/.npmrc

@@ -0,0 +1 @@
+better-sqlite3_local_prebuilds=../../prebuilds

app/entitlements.mac.plist

@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+  <dict>
+    <key>com.apple.security.cs.disable-library-validation</key><true/>
+    <key>com.apple.security.cs.allow-jit</key><true/>
+    <key>com.apple.security.cs.allow-unsigned-executable-memory</key><true/>
+    <key>com.apple.security.cs.allow-dyld-environment-variables</key><true/>
+    <key>com.apple.security.files.user-selected.read-only</key><true/>
+    <key>com.apple.security.files.user-selected.read-write</key><true/>
+  </dict>
+</plist>

app/package.json

@@ -1,41 +1,65 @@
 {
   "name": "dbgate",
-  "version": "3.9.5",
+  "version": "5.0.0-alpha.1",
   "private": true,
   "author": "Jan Prochazka <jenasoft.database@gmail.com>",
   "description": "Opensource database administration tool",
   "dependencies": {
-    "electron-log": "^4.3.1",
-    "electron-store": "^5.1.1",
-    "electron-updater": "^4.3.5"
+    "electron-log": "^4.4.1",
+    "electron-updater": "^4.6.1",
+    "lodash.clonedeepwith": "^4.5.0",
+    "patch-package": "^6.4.7"
   },
   "repository": {
     "type": "git",
     "url": "https://github.com/dbgate/dbgate.git"
   },
   "build": {
+    "artifactName": "dbgate-${version}-${os}_${arch}.${ext}",
     "appId": "org.dbgate",
+    "productName": "DbGate",
+    "afterSign": "electron-builder-notarize",
+    "asarUnpack": "**/*.node",
     "mac": {
       "category": "database",
-      "icon": "icon512.png",
-      "artifactName": "dbgate-mac-${version}.${ext}",
+      "icon": "icon512-mac.png",
+      "hardenedRuntime": true,
+      "entitlements": "entitlements.mac.plist",
+      "entitlementsInherit": "entitlements.mac.plist",
+      "publish": [
+        "github"
+      ],
+      "target": {
+        "target": "default",
+        "arch": [
+          "universal",
+          "x64"
+        ]
+      }
+    },
+    "linux": {
+      "target": [
+        "deb",
+        "snap",
+        {
+          "target": "AppImage",
+          "arch": [
+            "x64",
+            "armv7l",
+            "arm64"
+          ]
+        }
+      ],
+      "icon": "icons/",
+      "category": "Development",
+      "synopsis": "Database manager for SQL Server, MySQL, PostgreSQL, MongoDB and SQLite",
       "publish": [
         "github"
       ]
     },
-    "linux": {
-      "target": [
-        "AppImage",
-        "deb",
-        "snap"
-      ],
-      "icon": "icon.png",
-      "artifactName": "dbgate-linux-${version}.${ext}",
-      "category": "Development",
-      "synopsis": "Database administration tool for MS SQL, MySQL and PostgreSQL",
-      "publish": [
-        "github"
-      ]
+    "appImage": {
+      "license": "./LICENSE",
+      "category": "Development"
     },
     "snap": {
       "publish": [
@@ -48,9 +72,21 @@
     },
     "win": {
       "target": [
-        "nsis"
+        {
+          "target": "nsis",
+          "arch": [
+            "x64",
+            "arm64"
+          ]
+        },
+        {
+          "target": "zip",
+          "arch": [
+            "x64",
+            "arm64"
+          ]
+        }
       ],
-      "artifactName": "dbgate-windows-${version}.${ext}",
       "icon": "icon.ico",
       "publish": [
         "github"
@@ -65,22 +101,26 @@
   },
   "homepage": "./",
   "scripts": {
-    "start": "cross-env ELECTRON_START_URL=http://localhost:5000 electron .",
+    "start": "cross-env ELECTRON_START_URL=http://localhost:5001 DEVMODE=1 electron .",
     "start:local": "cross-env electron .",
     "dist": "electron-builder",
-    "build": "cd ../packages/api && yarn build && cd ../web && yarn build:app && cd ../../app && yarn dist",
-    "build:local": "cd ../packages/api && yarn build && cd ../web && yarn build:app && cd ../../app && yarn predist",
-    "postinstall": "electron-builder install-app-deps",
-    "predist": "copyfiles ../packages/api/dist/* packages && copyfiles \"../packages/web/build/*\" packages && copyfiles \"../packages/web/build/**/*\" packages"
+    "build": "cd ../packages/api && yarn build && cd ../web && yarn build && cd ../../app && yarn dist",
+    "build:local": "cd ../packages/api && yarn build && cd ../web && yarn build && cd ../../app && yarn predist",
+    "postinstall": "yarn rebuild && patch-package",
+    "rebuild": "electron-builder install-app-deps",
+    "predist": "copyfiles ../packages/api/dist/* packages && copyfiles \"../packages/web/public/*\" packages && copyfiles \"../packages/web/public/**/*\" packages && copyfiles --up 3 \"../plugins/dist/**/*\" packages/plugins"
   },
   "main": "src/electron.js",
   "devDependencies": {
     "copyfiles": "^2.2.0",
     "cross-env": "^6.0.3",
-    "electron": "11.2.3",
-    "electron-builder": "22.9.1"
+    "electron": "17.4.10",
+    "electron-builder": "23.1.0",
+    "electron-builder-notarize": "^1.5.1"
   },
   "optionalDependencies": {
-    "msnodesqlv8": "^2.0.10"
+    "better-sqlite3": "7.6.2",
+    "msnodesqlv8": "^2.6.0",
+    "oracledb": "^5.5.0"
   }
-}
+}

app/src/electron.js

@@ -1,10 +1,12 @@
 const electron = require('electron');
 const os = require('os');
+const fs = require('fs');
+// const unhandled = require('electron-unhandled');
+// const { openNewGitHubIssue, debugInfo } = require('electron-util');
 const { Menu, ipcMain } = require('electron');
-const { fork } = require('child_process');
 const { autoUpdater } = require('electron-updater');
-const Store = require('electron-store');
 const log = require('electron-log');
+const _cloneDeepWith = require('lodash.clonedeepwith');
 
 // Module to control application life.
 const app = electron.app;
@@ -13,172 +15,293 @@ const BrowserWindow = electron.BrowserWindow;
 
 const path = require('path');
 const url = require('url');
+const mainMenuDefinition = require('./mainMenuDefinition');
+const { settings } = require('cluster');
 
-const store = new Store();
+// require('@electron/remote/main').initialize();
+
+const configRootPath = path.join(app.getPath('userData'), 'config-root.json');
+let initialConfig = {};
+let apiLoaded = false;
+let mainModule;
+// let getLogger;
+// let loadLogsContent;
+
+const isMac = () => os.platform() == 'darwin';
+
+// unhandled({
+//   showDialog: true,
+//   reportButton: error => {
+//     openNewGitHubIssue({
+//       user: 'dbgate',
+//       repo: 'dbgate',
+//       body: `PLEASE DELETE SENSITIVE INFO BEFORE POSTING ISSUE!!!\n\n\`\`\`\n${
+//         error.stack
+//       }\n\`\`\`\n\n---\n\n${debugInfo()}\n\n\`\`\`\n${loadLogsContent ? loadLogsContent(50) : ''}\n\`\`\``,
+//     });
+//   },
+//   logger: error => (getLogger ? getLogger('electron').fatal(error) : console.error(error)),
+// });
+
+try {
+  initialConfig = JSON.parse(fs.readFileSync(configRootPath, { encoding: 'utf-8' }));
+} catch (err) {
+  console.log('Error loading config-root:', err.message);
+  initialConfig = {};
+}
 
 // Keep a global reference of the window object, if you don't, the window will
 // be closed automatically when the JavaScript object is garbage collected.
 let mainWindow;
-let splashWindow;
 let mainMenu;
+let runCommandOnLoad = null;
 
 log.transports.file.level = 'debug';
 autoUpdater.logger = log;
 // TODO - create settings for this
 // appUpdater.channel = 'beta';
 
-function hideSplash() {
-  if (splashWindow) {
-    splashWindow.destroy();
-    splashWindow = null;
+let commands = {};
+
+function formatKeyText(keyText) {
+  if (!keyText) {
+    return keyText;
   }
-  mainWindow.show();
+  if (os.platform() == 'darwin') {
+    return keyText.replace('CtrlOrCommand+', 'Command+');
+  }
+  return keyText.replace('CtrlOrCommand+', 'Ctrl+');
+}
+
+function commandItem(item) {
+  const id = item.command;
+  const command = commands[id];
+  if (item.skipInApp) {
+    return { skip: true };
+  }
+  return {
+    id,
+    label: command ? command.menuName || command.toolbarName || command.name : id,
+    accelerator: formatKeyText(command ? command.keyText : undefined),
+    enabled: command ? command.enabled : false,
+    click() {
+      if (mainWindow) {
+        mainWindow.webContents.send('run-command', id);
+      } else {
+        runCommandOnLoad = id;
+        createWindow();
+      }
+    },
+  };
 }
 
 function buildMenu() {
-  const template = [
-    {
-      label: 'File',
-      submenu: [
-        {
-          label: 'Connect to database',
-          click() {
-            mainWindow.webContents.executeJavaScript(`dbgate_createNewConnection()`);
-          },
-        },
-        {
-          label: 'Open file',
-          click() {
-            mainWindow.webContents.executeJavaScript(`dbgate_openFile()`);
-          },
-        },
-        {
-          label: 'Save',
-          click() {
-            mainWindow.webContents.executeJavaScript(`dbgate_tabCommand('save')`);
-          },
-          accelerator: 'Ctrl+S',
-          id: 'save',
-        },
-        {
-          label: 'Save As',
-          click() {
-            mainWindow.webContents.executeJavaScript(`dbgate_tabCommand('saveAs')`);
-          },
-          accelerator: 'Ctrl+Shift+S',
-          id: 'saveAs',
-        },
-        { type: 'separator' },
-        { role: 'close' },
-      ],
-    },
-    {
-      label: 'Window',
-      submenu: [
-        {
-          label: 'New query',
-          click() {
-            mainWindow.webContents.executeJavaScript(`dbgate_newQuery()`);
-          },
-        },
-        { type: 'separator' },
-        {
-          label: 'Close all tabs',
-          click() {
-            mainWindow.webContents.executeJavaScript('dbgate_closeAll()');
-          },
-        },
-        { role: 'minimize' },
-      ],
-    },
+  let template = _cloneDeepWith(mainMenuDefinition({ editMenu: true }), item => {
+    if (item.divider) {
+      return { type: 'separator' };
+    }
 
-    // {
-    //   label: 'Edit',
-    //   submenu: [
-    //     { role: 'undo' },
-    //     { role: 'redo' },
-    //     { type: 'separator' },
-    //     { role: 'cut' },
-    //     { role: 'copy' },
-    //     { role: 'paste' },
-    //   ],
-    // },
-    {
-      label: 'View',
-      submenu: [
-        { role: 'reload' },
-        { role: 'forcereload' },
-        { role: 'toggledevtools' },
-        { type: 'separator' },
-        { role: 'resetzoom' },
-        { role: 'zoomin' },
-        { role: 'zoomout' },
-        { type: 'separator' },
-        { role: 'togglefullscreen' },
-      ],
-    },
-    {
-      role: 'help',
-      submenu: [
-        {
-          label: 'dbgate.org',
-          click() {
-            require('electron').shell.openExternal('https://dbgate.org');
-          },
-        },
-        {
-          label: 'DbGate on GitHub',
-          click() {
-            require('electron').shell.openExternal('https://github.com/dbgate/dbgate');
-          },
-        },
-        {
-          label: 'DbGate on docker hub',
-          click() {
-            require('electron').shell.openExternal('https://hub.docker.com/r/dbgate/dbgate');
-          },
-        },
-        {
-          label: 'Report problem or feature request',
-          click() {
-            require('electron').shell.openExternal('https://github.com/dbgate/dbgate/issues/new');
-          },
-        },
-        {
-          label: 'About',
-          click() {
-            mainWindow.webContents.executeJavaScript(`dbgate_showAbout()`);
-          },
-        },
-      ],
-    },
-  ];
+    if (item.command) {
+      return commandItem(item);
+    }
+  });
+
+  template = _cloneDeepWith(template, item => {
+    if (Array.isArray(item) && item.find(x => x.skip)) {
+      return item.filter(x => x && !x.skip);
+    }
+  });
+
+  if (isMac()) {
+    template = [
+      {
+        label: 'DbGate',
+        submenu: [
+          commandItem({ command: 'about.show' }),
+          { role: 'services' },
+          { role: 'hide' },
+          { role: 'hideOthers' },
+          { role: 'unhide' },
+          { role: 'quit' },
+        ],
+      },
+      ...template,
+    ];
+  }
 
   return Menu.buildFromTemplate(template);
 }
 
-ipcMain.on('update-menu', async (event, arg) => {
-  const commands = await mainWindow.webContents.executeJavaScript(`dbgate_getCurrentTabCommands()`);
-  mainMenu.getMenuItemById('save').enabled = !!commands.save;
-  mainMenu.getMenuItemById('saveAs').enabled = !!commands.saveAs;
+ipcMain.on('update-commands', async (event, arg) => {
+  commands = JSON.parse(arg);
+  for (const key of Object.keys(commands)) {
+    const menu = mainMenu.getMenuItemById(key);
+    if (!menu) continue;
+    const command = commands[key];
+
+    // rebuild menu
+    if (menu.label != command.text || menu.accelerator != command.keyText) {
+      mainMenu = buildMenu();
+
+      Menu.setApplicationMenu(mainMenu);
+      // mainWindow.setMenu(mainMenu);
+      return;
+    }
+
+    menu.enabled = command.enabled;
+  }
+});
+ipcMain.on('quit-app', async (event, arg) => {
+  if (isMac()) {
+    app.quit();
+  } else {
+    mainWindow.close();
+  }
+});
+ipcMain.on('set-title', async (event, arg) => {
+  mainWindow.setTitle(arg);
+});
+ipcMain.on('open-link', async (event, arg) => {
+  electron.shell.openExternal(arg);
+});
+ipcMain.on('open-dev-tools', () => {
+  mainWindow.webContents.openDevTools();
+});
+ipcMain.on('app-started', async (event, arg) => {
+  if (runCommandOnLoad) {
+    mainWindow.webContents.send('run-command', runCommandOnLoad);
+    runCommandOnLoad = null;
+  }
+
+  if (initialConfig['winIsMaximized']) {
+    mainWindow.webContents.send('setIsMaximized', true);
+  }
+});
+ipcMain.on('window-action', async (event, arg) => {
+  if (!mainWindow) {
+    return;
+  }
+  switch (arg) {
+    case 'minimize':
+      mainWindow.minimize();
+      break;
+    case 'maximize':
+      if (mainWindow.isMaximized()) {
+        mainWindow.unmaximize();
+        mainWindow.webContents.send('setIsMaximized', false);
+      } else {
+        mainWindow.maximize();
+        mainWindow.webContents.send('setIsMaximized', true);
+      }
+      break;
+    case 'close':
+      mainWindow.close();
+      break;
+    case 'fullscreen-on':
+      mainWindow.setFullScreen(true);
+      break;
+    case 'fullscreen-off':
+      mainWindow.setFullScreen(false);
+      break;
+    case 'devtools':
+      mainWindow.webContents.toggleDevTools();
+      break;
+    case 'reload':
+      mainWindow.webContents.reloadIgnoringCache();
+      break;
+    case 'zoomin':
+      mainWindow.webContents.zoomLevel += 0.5;
+      break;
+    case 'zoomout':
+      mainWindow.webContents.zoomLevel -= 0.5;
+      break;
+    case 'zoomreset':
+      mainWindow.webContents.zoomLevel = 0;
+      break;
+
+    // edit
+    case 'undo':
+      mainWindow.webContents.undo();
+      break;
+    case 'redo':
+      mainWindow.webContents.redo();
+      break;
+    case 'cut':
+      mainWindow.webContents.cut();
+      break;
+    case 'copy':
+      mainWindow.webContents.copy();
+      break;
+    case 'paste':
+      mainWindow.webContents.paste();
+      break;
+  }
 });
 
+ipcMain.handle('showOpenDialog', async (event, options) => {
+  const res = electron.dialog.showOpenDialogSync(mainWindow, options);
+  return res;
+});
+ipcMain.handle('showSaveDialog', async (event, options) => {
+  const res = electron.dialog.showSaveDialogSync(mainWindow, options);
+  return res;
+});
+ipcMain.handle('showItemInFolder', async (event, path) => {
+  electron.shell.showItemInFolder(path);
+});
+ipcMain.handle('openExternal', async (event, url) => {
+  electron.shell.openExternal(url);
+});
+
+function fillMissingSettings(value) {
+  const res = {
+    ...value,
+  };
+  if (value['app.useNativeMenu'] !== true && value['app.useNativeMenu'] !== false) {
+    res['app.useNativeMenu'] = false;
+    // res['app.useNativeMenu'] = os.platform() == 'darwin' ? true : false;
+  }
+  return res;
+}
+
 function createWindow() {
-  const bounds = store.get('winBounds');
+  let settingsJson = {};
+  try {
+    const datadir = path.join(os.homedir(), '.dbgate');
+    settingsJson = fillMissingSettings(
+      JSON.parse(fs.readFileSync(path.join(datadir, 'settings.json'), { encoding: 'utf-8' }))
+    );
+  } catch (err) {
+    console.log('Error loading settings.json:', err.message);
+    settingsJson = fillMissingSettings({});
+  }
+
+  const bounds = initialConfig['winBounds'];
+  useNativeMenu = settingsJson['app.useNativeMenu'];
 
   mainWindow = new BrowserWindow({
     width: 1200,
     height: 800,
     title: 'DbGate',
+    frame: useNativeMenu,
+    titleBarStyle: useNativeMenu ? undefined : 'hidden',
     ...bounds,
     icon: os.platform() == 'win32' ? 'icon.ico' : path.resolve(__dirname, '../icon.png'),
-    show: false,
+    partition: 'persist:dbgate',
     webPreferences: {
       nodeIntegration: true,
-      enableRemoteModule: true,
+      contextIsolation: false,
+      spellcheck: false,
     },
   });
 
+  if (initialConfig['winIsMaximized']) {
+    mainWindow.maximize();
+  }
+  if (settingsJson['app.fullscreen']) {
+    mainWindow.setFullScreen(true);
+  }
+
   mainMenu = buildMenu();
   mainWindow.setMenu(mainMenu);
 
@@ -186,59 +309,59 @@ function createWindow() {
     const startUrl =
       process.env.ELECTRON_START_URL ||
       url.format({
-        pathname: path.join(__dirname, '../packages/web/build/index.html'),
+        pathname: path.join(__dirname, '../packages/web/public/index.html'),
         protocol: 'file:',
         slashes: true,
       });
-    mainWindow.webContents.on('did-finish-load', function () {
-      hideSplash();
-    });
     mainWindow.on('close', () => {
-      store.set('winBounds', mainWindow.getBounds());
+      try {
+        fs.writeFileSync(
+          configRootPath,
+          JSON.stringify({
+            winBounds: mainWindow.getBounds(),
+            winIsMaximized: mainWindow.isMaximized(),
+          }),
+          'utf-8'
+        );
+      } catch (err) {
+        console.log('Error saving config-root:', err.message);
+      }
     });
     mainWindow.loadURL(startUrl);
     if (os.platform() == 'linux') {
       mainWindow.setIcon(path.resolve(__dirname, '../icon.png'));
     }
+    // mainWindow.webContents.toggleDevTools();
   }
 
-  splashWindow = new BrowserWindow({
-    width: 300,
-    height: 120,
-    transparent: true,
-    frame: false,
-  });
-  splashWindow.loadURL(
-    url.format({
-      pathname: path.join(__dirname, '../packages/web/build/splash.html'),
-      protocol: 'file:',
-      slashes: true,
-    })
-  );
+  if (!apiLoaded) {
+    const apiPackage = path.join(
+      __dirname,
+      process.env.DEVMODE ? '../../packages/api/src/index' : '../packages/api/dist/bundle.js'
+    );
 
-  if (process.env.ELECTRON_START_URL) {
-    loadMainWindow();
-  } else {
-    const apiProcess = fork(path.join(__dirname, '../packages/api/dist/bundle.js'), [
-      '--dynport',
-      '--native-modules',
-      path.join(__dirname, 'nativeModules'),
-      // '../../../src/nativeModules'
-    ]);
-    apiProcess.on('message', msg => {
-      if (msg.msgtype == 'listening') {
-        const { port } = msg;
-        global['port'] = port;
-        loadMainWindow();
-      }
-    });
+    global.API_PACKAGE = apiPackage;
+    global.NATIVE_MODULES = path.join(__dirname, 'nativeModules');
+
+    // console.log('global.API_PACKAGE', global.API_PACKAGE);
+    const api = require(apiPackage);
+    // console.log(
+    //   'REQUIRED',
+    //   path.resolve(
+    //     path.join(__dirname, process.env.DEVMODE ? '../../packages/api/src/index' : '../packages/api/dist/bundle.js')
+    //   )
+    // );
+    api.configureLogger();
+    const main = api.getMainModule();
+    main.useAllControllers(null, electron);
+    mainModule = main;
+    // getLogger = api.getLogger;
+    // loadLogsContent = api.loadLogsContent;
+    apiLoaded = true;
   }
+  mainModule.setElectronSender(mainWindow.webContents);
 
-  // and load the index.html of the app.
-  // mainWindow.loadURL('http://localhost:3000');
-
-  // Open the DevTools.
-  // mainWindow.webContents.openDevTools();
+  loadMainWindow();
 
   // Emitted when the window is closed.
   mainWindow.on('closed', function () {
@@ -246,11 +369,14 @@ function createWindow() {
     // in an array if your app supports multi windows, this is the time
     // when you should delete the corresponding element.
     mainWindow = null;
+    mainModule.setElectronSender(null);
   });
 }
 
 function onAppReady() {
-  autoUpdater.checkForUpdatesAndNotify();
+  if (!process.env.DEVMODE) {
+    autoUpdater.checkForUpdatesAndNotify();
+  }
   createWindow();
 }
 
@@ -263,7 +389,7 @@ app.on('ready', onAppReady);
 app.on('window-all-closed', function () {
   // On OS X it is common for applications and their menu bar
   // to stay active until the user quits explicitly with Cmd + Q
-  if (process.platform !== 'darwin') {
+  if (!isMac()) {
     app.quit();
   }
 });

app/src/mainMenuDefinition.js

@@ -0,0 +1,107 @@
+module.exports = ({ editMenu }) => [
+  {
+    label: 'File',
+    submenu: [
+      { command: 'new.connection', hideDisabled: true },
+      { command: 'new.sqliteDatabase', hideDisabled: true },
+      { divider: true },
+      { command: 'new.query', hideDisabled: true },
+      { command: 'new.queryDesign', hideDisabled: true },
+      { command: 'new.diagram', hideDisabled: true },
+      { command: 'new.perspective', hideDisabled: true },
+      { command: 'new.freetable', hideDisabled: true },
+      { command: 'new.shell', hideDisabled: true },
+      { command: 'new.jsonl', hideDisabled: true },
+      { divider: true },
+      { command: 'file.open', hideDisabled: true },
+      { command: 'file.openArchive', hideDisabled: true },
+      { divider: true },
+      { command: 'group.save', hideDisabled: true },
+      { command: 'group.saveAs', hideDisabled: true },
+      { divider: true },
+      { command: 'file.exit', hideDisabled: true },
+      { command: 'app.logout', hideDisabled: true, skipInApp: true },
+      { command: 'app.disconnect', hideDisabled: true, skipInApp: true },
+    ],
+  },
+  {
+    label: 'Window',
+    submenu: [
+      { command: 'tabs.closeTab', hideDisabled: false },
+      { command: 'tabs.closeAll', hideDisabled: false },
+      { command: 'tabs.closeTabsWithCurrentDb', hideDisabled: false },
+      { command: 'tabs.closeTabsButCurrentDb', hideDisabled: false },
+      { divider: true },
+      { command: 'app.zoomIn', hideDisabled: true },
+      { command: 'app.zoomOut', hideDisabled: true },
+      { command: 'app.zoomReset', hideDisabled: true },
+    ],
+  },
+
+  editMenu
+    ? {
+        label: 'Edit',
+        submenu: [
+          { command: 'edit.undo' },
+          { command: 'edit.redo' },
+          { divider: true },
+          { command: 'edit.cut' },
+          { command: 'edit.copy' },
+          { command: 'edit.paste' },
+        ],
+      }
+    : null,
+
+  // {
+  //   label: 'Edit',
+  //   submenu: [
+  //     { role: 'undo' },
+  //     { role: 'redo' },
+  //     { type: 'separator' },
+  //     { role: 'cut' },
+  //     { role: 'copy' },
+  //     { role: 'paste' },
+  //   ],
+  // },
+  {
+    label: 'View',
+    submenu: [
+      { command: 'app.reload', hideDisabled: true },
+      { command: 'app.toggleDevTools', hideDisabled: true },
+      { command: 'app.toggleFullScreen', hideDisabled: true },
+      { command: 'app.minimize', hideDisabled: true },
+      { command: 'toggle.sidebar' },
+      { divider: true },
+      { command: 'theme.changeTheme', hideDisabled: true },
+      { command: 'settings.show' },
+    ],
+  },
+  {
+    label: 'Tools',
+    submenu: [
+      { command: 'database.search', hideDisabled: true },
+      { command: 'commandPalette.show', hideDisabled: true },
+      { command: 'database.switch', hideDisabled: true },
+      { divider: true },
+      { command: 'sql.generator', hideDisabled: true },
+      { command: 'file.import', hideDisabled: true },
+      { command: 'new.modelCompare', hideDisabled: true },
+      { divider: true },
+      { command: 'folder.showLogs', hideDisabled: true },
+      { command: 'folder.showData', hideDisabled: true },
+    ],
+  },
+  {
+    label: 'Help',
+    submenu: [
+      { command: 'app.openDocs', hideDisabled: true },
+      { command: 'app.openWeb', hideDisabled: true },
+      { command: 'app.openIssue', hideDisabled: true },
+      { command: 'app.openSponsoring', hideDisabled: true },
+      { divider: true },
+      { command: 'settings.commands', hideDisabled: true },
+      { command: 'tabs.changelog', hideDisabled: true },
+      { command: 'about.show', hideDisabled: true },
+    ],
+  },
+];

docker-compose.yaml

@@ -0,0 +1,54 @@
+# this compose file is for testing purposes only
+# use it for testing docker containsers built on local machine
+version: "3"
+services:
+  dbgate:
+    build: docker
+    # image: dbgate/dbgate:beta-alpine
+    # image: dbgate/dbgate:alpine
+    # image: dbgate/dbgate:beta
+    restart: always
+    ports:
+      - 3100:3000
+    # volumes:
+    #   - /home/jena/dbgate-data:/root/dbgate-data
+
+    volumes:
+      - dbgate-data:/root/.dbgate
+    
+    # environment:
+    #   WEB_ROOT: /dbgate
+
+      # CONNECTIONS: mssql
+      # LABEL_mssql: MS Sql
+      # SERVER_mssql: mssql
+      # USER_mssql: sa
+      # PORT_mssql: 1433
+      # PASSWORD_mssql: Pwd2020Db
+      # ENGINE_mssql: mssql@dbgate-plugin-mssql
+  # proxy:
+  #   # image: nginx
+  #   build: test/nginx
+  #   ports:
+  #     - 8082:80
+
+    # volumes:
+    #   - /home/jena/test/chinook:/mnt/sqt
+    # environment:
+    #   CONNECTIONS: sqlite
+
+    #   LABEL_sqlite: sqt
+    #   FILE_sqlite: /mnt/sqt/Chinook.db
+    #   ENGINE_sqlite: sqlite@dbgate-plugin-sqlite
+
+  # mssql:
+  #   image: mcr.microsoft.com/mssql/server
+  #   restart: always
+  #   environment:
+  #     - ACCEPT_EULA=Y
+  #     - SA_PASSWORD=Pwd2020Db
+  #     - MSSQL_PID=Express
+
+volumes:
+  dbgate-data:
+    driver: local

docker/.gitignore

@@ -0,0 +1,2 @@
+package.json
+yarn.lock

docker/Dockerfile

@@ -1,9 +1,18 @@
-FROM node:12-alpine
+FROM node:14
+
+RUN apt-get update && apt-get install -y \
+    iputils-ping \
+    iproute2 \
+ && rm -rf /var/lib/apt/lists/*
 
 WORKDIR /home/dbgate-docker
 
 COPY . .
 
+RUN ["chmod", "+x", "/home/dbgate-docker/entrypoint.sh"]
+
 WORKDIR /home/dbgate-docker
 EXPOSE 3000
-CMD node bundle.js
+VOLUME /root/.dbgate
+
+CMD ["/home/dbgate-docker/entrypoint.sh"]

docker/Dockerfile-alpine

@@ -0,0 +1,17 @@
+FROM node:14-alpine
+
+WORKDIR /home/dbgate-docker
+
+RUN apk --no-cache upgrade \
+ && apk --no-cache add  \
+    iputils
+
+COPY . .
+
+RUN ["chmod", "+x", "/home/dbgate-docker/entrypoint.sh"]
+
+WORKDIR /home/dbgate-docker
+EXPOSE 3000
+VOLUME /root/.dbgate
+
+CMD ["/home/dbgate-docker/entrypoint.sh"]

docker/entrypoint.sh

@@ -0,0 +1,11 @@
+#!/bin/sh
+
+HOST_DOMAIN="dockerhost"
+ping -q -c1 $HOST_DOMAIN > /dev/null 2>&1
+if [ $? != 0 ]
+then
+  HOST_IP=$(ip route | awk 'NR==1 {print $3}')
+  echo "$HOST_IP $HOST_DOMAIN" >> /etc/hosts
+fi
+
+exec node bundle.js --listen-api

fillNativeModules.js

@@ -2,12 +2,15 @@ const fs = require('fs');
 
 let fillContent = '';
 
-// if (!process.argv.includes('--electron')) {
 if (process.platform == 'win32') {
   fillContent += `content.msnodesqlv8 = () => require('msnodesqlv8');`;
 }
+if (process.arch != 'arm64') {
+  fillContent += `content.oracledb = () => require('oracledb');`;
+}
+fillContent += `content['better-sqlite3'] = () => require('better-sqlite3');`;
 
-const getContent = (empty) => `
+const getContent = empty => `
 // this file is generated automatically by script fillNativeModules.js, do not edit it manually
 const content = {};
 

fillPackagedPlugins.js

@@ -0,0 +1,23 @@
+const fs = require('fs');
+const path = require('path');
+
+function load() {
+  const plugins = {};
+
+  for (const packageName of fs.readdirSync('plugins')) {
+    if (!packageName.startsWith('dbgate-plugin-')) continue;
+    const dir = path.join('plugins', packageName);
+    const frontend = fs.readFileSync(path.join(dir, 'dist', 'frontend.js'), 'utf-8');
+    const readme = fs.readFileSync(path.join(dir, 'README.md'), 'utf-8');
+    const manifest = JSON.parse(fs.readFileSync(path.join(dir, 'package.json'), 'utf-8'));
+    plugins[packageName] = {
+      manifest,
+      frontend,
+      readme,
+    };
+  }
+
+  return plugins;
+}
+
+fs.writeFileSync('packages/api/src/packagedPluginsContent.js', `module.exports = () => (${JSON.stringify(load())});`);

integration-tests/.gitignore

@@ -0,0 +1 @@
+dbtemp

integration-tests/__tests__/alter-database.spec.js

@@ -0,0 +1,68 @@
+const stableStringify = require('json-stable-stringify');
+const _ = require('lodash');
+const fp = require('lodash/fp');
+const uuidv1 = require('uuid/v1');
+const { testWrapper } = require('../tools');
+const engines = require('../engines');
+const { getAlterDatabaseScript, extendDatabaseInfo, generateDbPairingId } = require('dbgate-tools');
+
+function flatSource() {
+  return _.flatten(
+    engines.map(engine => (engine.objects || []).map(object => [engine.label, object.type, object, engine]))
+  );
+}
+
+async function testDatabaseDiff(conn, driver, mangle, createObject = null) {
+  await driver.query(conn, `create table t1 (id int not null primary key)`);
+
+  await driver.query(
+    conn,
+    `create table t2 (
+    id int not null primary key, 
+    t1_id int null references t1(id)
+  )`
+  );
+
+  if (createObject) await driver.query(conn, createObject);
+
+  const structure1 = generateDbPairingId(extendDatabaseInfo(await driver.analyseFull(conn)));
+  let structure2 = _.cloneDeep(structure1);
+  mangle(structure2);
+  structure2 = extendDatabaseInfo(structure2);
+
+  const { sql } = getAlterDatabaseScript(structure1, structure2, {}, structure1, structure2, driver);
+  console.log('RUNNING ALTER SQL', driver.engine, ':', sql);
+
+  await driver.script(conn, sql);
+
+  const structure2Real = extendDatabaseInfo(await driver.analyseFull(conn));
+
+  expect(structure2Real.tables.length).toEqual(structure2.tables.length);
+  return structure2Real;
+}
+
+describe('Alter database', () => {
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Drop referenced table - %s',
+    testWrapper(async (conn, driver, engine) => {
+      await testDatabaseDiff(conn, driver, db => {
+        _.remove(db.tables, x => x.pureName == 't1');
+      });
+    })
+  );
+
+  test.each(flatSource())(
+    'Drop object - %s - %s',
+    testWrapper(async (conn, driver, type, object, engine) => {
+      const db = await testDatabaseDiff(
+        conn,
+        driver,
+        db => {
+          _.remove(db[type], x => x.pureName == 'obj1');
+        },
+        object.create1
+      );
+      expect(db[type].length).toEqual(0);
+    })
+  );
+});

integration-tests/__tests__/alter-table.spec.js

@@ -0,0 +1,124 @@
+const stableStringify = require('json-stable-stringify');
+const _ = require('lodash');
+const fp = require('lodash/fp');
+const uuidv1 = require('uuid/v1');
+const { testWrapper } = require('../tools');
+const engines = require('../engines');
+const { getAlterTableScript, extendDatabaseInfo, generateDbPairingId } = require('dbgate-tools');
+
+function pickImportantTableInfo(table) {
+  return {
+    pureName: table.pureName,
+    columns: table.columns
+      .filter(x => x.columnName != 'rowid')
+      .map(fp.pick(['columnName', 'notNull', 'autoIncrement'])),
+  };
+}
+
+function checkTableStructure(t1, t2) {
+  // expect(t1.pureName).toEqual(t2.pureName)
+  expect(pickImportantTableInfo(t1)).toEqual(pickImportantTableInfo(t2));
+}
+
+async function testTableDiff(conn, driver, mangle) {
+  await driver.query(conn, `create table t0 (id int not null primary key)`);
+
+  await driver.query(
+    conn,
+    `create table t1 (
+    col_pk int not null primary key, 
+    col_std int null, 
+    col_def int null default 12,
+    col_fk int null references t0(id),
+    col_idx int null,
+    col_uq int null unique,
+    col_ref int null unique
+  )`
+  );
+
+  await driver.query(conn, `create index idx1 on t1(col_idx)`);
+
+  await driver.query(conn, `create table t2 (id int not null primary key, fkval int null references t1(col_ref))`);
+
+  const tget = x => x.tables.find(y => y.pureName == 't1');
+  const structure1 = generateDbPairingId(extendDatabaseInfo(await driver.analyseFull(conn)));
+  let structure2 = _.cloneDeep(structure1);
+  mangle(tget(structure2));
+  structure2 = extendDatabaseInfo(structure2);
+
+  const { sql } = getAlterTableScript(tget(structure1), tget(structure2), {}, structure1, structure2, driver);
+  console.log('RUNNING ALTER SQL', driver.engine, ':', sql);
+
+  await driver.script(conn, sql);
+
+  const structure2Real = extendDatabaseInfo(await driver.analyseFull(conn));
+
+  checkTableStructure(tget(structure2Real), tget(structure2));
+  // expect(stableStringify(structure2)).toEqual(stableStringify(structure2Real));
+}
+
+const TESTED_COLUMNS = ['col_pk', 'col_std', 'col_def', 'col_fk', 'col_ref', 'col_idx', 'col_uq'];
+// const TESTED_COLUMNS = ['col_pk'];
+// const TESTED_COLUMNS = ['col_idx'];
+// const TESTED_COLUMNS = ['col_def'];
+// const TESTED_COLUMNS = ['col_std'];
+// const TESTED_COLUMNS = ['col_ref'];
+
+function engines_columns_source() {
+  return _.flatten(engines.map(engine => TESTED_COLUMNS.map(column => [engine.label, column, engine])));
+}
+
+describe('Alter table', () => {
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Add column - %s',
+    testWrapper(async (conn, driver, engine) => {
+      await testTableDiff(conn, driver, tbl => {
+        tbl.columns.push({
+          columnName: 'added',
+          dataType: 'int',
+          pairingId: uuidv1(),
+          notNull: false,
+          autoIncrement: false,
+        });
+      });
+    })
+  );
+
+  test.each(engines_columns_source())(
+    'Drop column - %s - %s',
+    testWrapper(async (conn, driver, column, engine) => {
+      await testTableDiff(conn, driver, tbl => (tbl.columns = tbl.columns.filter(x => x.columnName != column)));
+    })
+  );
+
+  test.each(engines_columns_source())(
+    'Change nullability - %s - %s',
+    testWrapper(async (conn, driver, column, engine) => {
+      await testTableDiff(
+        conn,
+        driver,
+        tbl => (tbl.columns = tbl.columns.map(x => (x.columnName == column ? { ...x, notNull: true } : x)))
+      );
+    })
+  );
+
+  test.each(engines_columns_source())(
+    'Rename column - %s - %s',
+    testWrapper(async (conn, driver, column, engine) => {
+      await testTableDiff(
+        conn,
+        driver,
+        tbl => (tbl.columns = tbl.columns.map(x => (x.columnName == column ? { ...x, columnName: 'col_renamed' } : x)))
+      );
+    })
+  );
+
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Drop index - %s',
+    testWrapper(async (conn, driver, engine) => {
+      await testTableDiff(conn, driver, tbl => {
+        tbl.indexes = [];
+      });
+    })
+  );
+});

integration-tests/__tests__/data-duplicator.spec.js

@@ -0,0 +1,94 @@
+const engines = require('../engines');
+const stream = require('stream');
+const { testWrapper } = require('../tools');
+const dataDuplicator = require('dbgate-api/src/shell/dataDuplicator');
+const { runCommandOnDriver } = require('dbgate-tools');
+
+describe('Data duplicator', () => {
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Insert simple data - %s',
+    testWrapper(async (conn, driver, engine) => {
+      runCommandOnDriver(conn, driver, dmp =>
+        dmp.createTable({
+          pureName: 't1',
+          columns: [
+            { columnName: 'id', dataType: 'int', autoIncrement: true, notNull: true },
+            { columnName: 'val', dataType: 'varchar(50)' },
+          ],
+          primaryKey: {
+            columns: [{ columnName: 'id' }],
+          },
+        })
+      );
+      runCommandOnDriver(conn, driver, dmp =>
+        dmp.createTable({
+          pureName: 't2',
+          columns: [
+            { columnName: 'id', dataType: 'int', autoIncrement: true, notNull: true },
+            { columnName: 'val', dataType: 'varchar(50)' },
+            { columnName: 'valfk', dataType: 'int', notNull: true },
+          ],
+          primaryKey: {
+            columns: [{ columnName: 'id' }],
+          },
+          foreignKeys: [{ refTableName: 't1', columns: [{ columnName: 'valfk', refColumnName: 'id' }] }],
+        })
+      );
+
+      const gett1 = () =>
+        stream.Readable.from([
+          { __isStreamHeader: true, __isDynamicStructure: true },
+          { id: 1, val: 'v1' },
+          { id: 2, val: 'v2' },
+          { id: 3, val: 'v3' },
+        ]);
+      const gett2 = () =>
+        stream.Readable.from([
+          { __isStreamHeader: true, __isDynamicStructure: true },
+          { id: 1, val: 'v1', valfk: 1 },
+          { id: 2, val: 'v2', valfk: 2 },
+          { id: 3, val: 'v3', valfk: 3 },
+        ]);
+
+      await dataDuplicator({
+        systemConnection: conn,
+        driver,
+        items: [
+          {
+            name: 't1',
+            operation: 'copy',
+            openStream: gett1,
+          },
+          {
+            name: 't2',
+            operation: 'copy',
+            openStream: gett2,
+          },
+        ],
+      });
+
+      await dataDuplicator({
+        systemConnection: conn,
+        driver,
+        items: [
+          {
+            name: 't1',
+            operation: 'copy',
+            openStream: gett1,
+          },
+          {
+            name: 't2',
+            operation: 'copy',
+            openStream: gett2,
+          },
+        ],
+      });
+
+      const res1 = await driver.query(conn, `select count(*) as cnt from t1`);
+      expect(res1.rows[0].cnt.toString()).toEqual('6');
+
+      const res2 = await driver.query(conn, `select count(*) as cnt from t2`);
+      expect(res2.rows[0].cnt.toString()).toEqual('6');
+    })
+  );
+});

integration-tests/__tests__/db-import.spec.js

@@ -0,0 +1,75 @@
+const engines = require('../engines');
+const stream = require('stream');
+const { testWrapper } = require('../tools');
+const tableWriter = require('dbgate-api/src/shell/tableWriter');
+const copyStream = require('dbgate-api/src/shell/copyStream');
+const fakeObjectReader = require('dbgate-api/src/shell/fakeObjectReader');
+
+function createImportStream() {
+  const pass = new stream.PassThrough({
+    objectMode: true,
+  });
+  pass.write({ columns: [{ columnName: 'id' }, { columnName: 'country' }], __isStreamHeader: true });
+  pass.write({ id: 1, country: 'Czechia' });
+  pass.write({ id: 2, country: 'Austria' });
+  pass.write({ country: 'Germany', id: 3 });
+  pass.write({ country: 'Romania', id: 4 });
+  pass.write({ country: 'Great Britain', id: 5 });
+  pass.write({ country: 'Bosna, Hecegovina', id: 6 });
+  pass.end();
+
+  return pass;
+}
+
+describe('DB Import', () => {
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Import one table - %s',
+    testWrapper(async (conn, driver, engine) => {
+      // const reader = await fakeObjectReader({ delay: 10 });
+      // const reader = await fakeObjectReader();
+      const reader = createImportStream();
+      const writer = await tableWriter({
+        systemConnection: conn,
+        driver,
+        pureName: 't1',
+        createIfNotExists: true,
+      });
+      await copyStream(reader, writer);
+
+      const res = await driver.query(conn, `select count(*) as cnt from t1`);
+      expect(res.rows[0].cnt.toString()).toEqual('6');
+    })
+  );
+
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Import two tables - %s',
+    testWrapper(async (conn, driver, engine) => {
+      // const reader = await fakeObjectReader({ delay: 10 });
+      // const reader = await fakeObjectReader();
+      const reader1 = createImportStream();
+      const writer1 = await tableWriter({
+        systemConnection: conn,
+        driver,
+        pureName: 't1',
+        createIfNotExists: true,
+      });
+      await copyStream(reader1, writer1);
+
+      const reader2 = createImportStream();
+      const writer2 = await tableWriter({
+        systemConnection: conn,
+        driver,
+        pureName: 't2',
+        createIfNotExists: true,
+      });
+      await copyStream(reader2, writer2);
+
+      const res1 = await driver.query(conn, `select count(*) as cnt from t1`);
+      expect(res1.rows[0].cnt.toString()).toEqual('6');
+
+      const res2 = await driver.query(conn, `select count(*) as cnt from t2`);
+      expect(res2.rows[0].cnt.toString()).toEqual('6');
+    })
+  );
+
+});

integration-tests/__tests__/deploy-database.spec.js

@@ -0,0 +1,329 @@
+/// TODO
+
+const { testWrapper } = require('../tools');
+const _ = require('lodash');
+const engines = require('../engines');
+const deployDb = require('dbgate-api/src/shell/deployDb');
+const { databaseInfoFromYamlModel } = require('dbgate-tools');
+const generateDeploySql = require('dbgate-api/src/shell/generateDeploySql');
+
+function checkStructure(structure, model) {
+  const expected = databaseInfoFromYamlModel(model);
+  expect(structure.tables.length).toEqual(expected.tables.length);
+
+  for (const [realTable, expectedTable] of _.zip(
+    _.sortBy(structure.tables, 'pureName'),
+    _.sortBy(expected.tables, 'pureName')
+  )) {
+    expect(realTable.columns.length).toBeGreaterThanOrEqual(expectedTable.columns.length);
+  }
+}
+
+async function testDatabaseDeploy(conn, driver, dbModelsYaml, testEmptyLastScript) {
+  let index = 0;
+  for (const loadedDbModel of dbModelsYaml) {
+    const { sql, isEmpty } = await generateDeploySql({
+      systemConnection: conn,
+      driver,
+      loadedDbModel,
+    });
+    console.debug('Generated deploy script:', sql);
+    expect(sql.toUpperCase().includes('DROP ')).toBeFalsy();
+
+    console.log('dbModelsYaml.length', dbModelsYaml.length, index);
+    if (testEmptyLastScript && index == dbModelsYaml.length - 1) {
+      expect(isEmpty).toBeTruthy();
+    }
+
+    await deployDb({
+      systemConnection: conn,
+      driver,
+      loadedDbModel,
+    });
+
+    index++;
+  }
+
+  const structure = await driver.analyseFull(conn);
+  checkStructure(structure, dbModelsYaml[dbModelsYaml.length - 1]);
+}
+
+describe('Deploy database', () => {
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Deploy database simple - %s',
+    testWrapper(async (conn, driver, engine) => {
+      await testDatabaseDeploy(conn, driver, [
+        [
+          {
+            name: 't1.table.yaml',
+            json: {
+              name: 't1',
+              columns: [{ name: 'id', type: 'int' }],
+              primaryKey: ['id'],
+            },
+          },
+        ],
+      ]);
+    })
+  );
+
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Deploy database simple twice - %s',
+    testWrapper(async (conn, driver, engine) => {
+      await testDatabaseDeploy(
+        conn,
+        driver,
+        [
+          [
+            {
+              name: 't1.table.yaml',
+              json: {
+                name: 't1',
+                columns: [{ name: 'id', type: 'int' }],
+                primaryKey: ['id'],
+              },
+            },
+          ],
+          [
+            {
+              name: 't1.table.yaml',
+              json: {
+                name: 't1',
+                columns: [{ name: 'id', type: 'int' }],
+                primaryKey: ['id'],
+              },
+            },
+          ],
+        ],
+        true
+      );
+    })
+  );
+
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Add column - %s',
+    testWrapper(async (conn, driver, engine) => {
+      await testDatabaseDeploy(conn, driver, [
+        [
+          {
+            name: 't1.table.yaml',
+            json: {
+              name: 't1',
+              columns: [{ name: 'id', type: 'int' }],
+              primaryKey: ['id'],
+            },
+          },
+        ],
+        [
+          {
+            name: 't1.table.yaml',
+            json: {
+              name: 't1',
+              columns: [
+                { name: 'id', type: 'int' },
+                { name: 'val', type: 'int' },
+              ],
+              primaryKey: ['id'],
+            },
+          },
+        ],
+      ]);
+    })
+  );
+
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Dont drop column - %s',
+    testWrapper(async (conn, driver, engine) => {
+      await testDatabaseDeploy(
+        conn,
+        driver,
+        [
+          [
+            {
+              name: 't1.table.yaml',
+              json: {
+                name: 't1',
+                columns: [
+                  { name: 'id', type: 'int' },
+                  { name: 'val', type: 'int' },
+                ],
+                primaryKey: ['id'],
+              },
+            },
+          ],
+          [
+            {
+              name: 't1.table.yaml',
+              json: {
+                name: 't1',
+                columns: [{ name: 'id', type: 'int' }],
+                primaryKey: ['id'],
+              },
+            },
+          ],
+        ],
+        true
+      );
+    })
+  );
+
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Foreign keys - %s',
+    testWrapper(async (conn, driver, engine) => {
+      await testDatabaseDeploy(
+        conn,
+        driver,
+        [
+          [
+            {
+              name: 't2.table.yaml',
+              json: {
+                name: 't2',
+                columns: [
+                  { name: 't2id', type: 'int' },
+                  { name: 't1id', type: 'int', references: 't1' },
+                ],
+                primaryKey: ['t2id'],
+              },
+            },
+            {
+              name: 't1.table.yaml',
+              json: {
+                name: 't1',
+                columns: [{ name: 't1id', type: 'int' }],
+                primaryKey: ['t1id'],
+              },
+            },
+          ],
+          [
+            {
+              name: 't2.table.yaml',
+              json: {
+                name: 't2',
+                columns: [
+                  { name: 't2id', type: 'int' },
+                  { name: 't1id', type: 'int', references: 't1' },
+                ],
+                primaryKey: ['t2id'],
+              },
+            },
+            {
+              name: 't1.table.yaml',
+              json: {
+                name: 't1',
+                columns: [{ name: 't1id', type: 'int' }],
+                primaryKey: ['t1id'],
+              },
+            },
+          ],
+        ],
+        true
+      );
+    })
+  );
+
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Deploy preloaded data - %s',
+    testWrapper(async (conn, driver, engine) => {
+      await testDatabaseDeploy(conn, driver, [
+        [
+          {
+            name: 't1.table.yaml',
+            json: {
+              name: 't1',
+              columns: [
+                { name: 'id', type: 'int' },
+                { name: 'value', type: 'int' },
+              ],
+              primaryKey: ['id'],
+              data: [
+                { id: 1, value: 1 },
+                { id: 2, value: 2 },
+                { id: 3, value: 3 },
+              ],
+            },
+          },
+        ],
+      ]);
+
+      const res = await driver.query(conn, `select count(*) as cnt from t1`);
+      expect(res.rows[0].cnt.toString()).toEqual('3');
+    })
+  );
+
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Deploy preloaded data - update - %s',
+    testWrapper(async (conn, driver, engine) => {
+      await testDatabaseDeploy(conn, driver, [
+        [
+          {
+            name: 't1.table.yaml',
+            json: {
+              name: 't1',
+              columns: [
+                { name: 'id', type: 'int' },
+                { name: 'val', type: 'int' },
+              ],
+              primaryKey: ['id'],
+              data: [
+                { id: 1, val: 1 },
+                { id: 2, val: 2 },
+                { id: 3, val: 3 },
+              ],
+            },
+          },
+        ],
+        [
+          {
+            name: 't1.table.yaml',
+            json: {
+              name: 't1',
+              columns: [
+                { name: 'id', type: 'int' },
+                { name: 'val', type: 'int' },
+              ],
+              primaryKey: ['id'],
+              data: [
+                { id: 1, val: 1 },
+                { id: 2, val: 5 },
+                { id: 3, val: 3 },
+              ],
+            },
+          },
+        ],
+      ]);
+
+      const res = await driver.query(conn, `select val from t1 where id = 2`);
+      expect(res.rows[0].val.toString()).toEqual('5');
+    })
+  );
+
+  test.each(engines.enginesPostgre.map(engine => [engine.label, engine]))(
+    'Current timestamp default value - %s',
+    testWrapper(async (conn, driver, engine) => {
+      await testDatabaseDeploy(conn, driver, [
+        [
+          {
+            name: 't1.table.yaml',
+            json: {
+              name: 't1',
+              columns: [
+                { name: 'id', type: 'int' },
+                {
+                  name: 'val',
+                  type: 'timestamp',
+                  default: 'current_timestamp',
+                },
+              ],
+              primaryKey: ['id'],
+            },
+          },
+        ],
+      ]);
+
+      await driver.query(conn, `insert into t1 (id) values (1)`);
+      const res = await driver.query(conn, ` select val from t1 where id = 1`);
+      expect(res.rows[0].val.toString().substring(0, 2)).toEqual('20');
+    })
+  );
+});

integration-tests/__tests__/object-analyse.spec.js

@@ -0,0 +1,89 @@
+const { testWrapper } = require('../tools');
+const engines = require('../engines');
+const _ = require('lodash');
+
+const initSql = ['CREATE TABLE t1 (id int)', 'CREATE TABLE t2 (id int)'];
+
+function flatSource() {
+  return _.flatten(
+    engines.map(engine => (engine.objects || []).map(object => [engine.label, object.type, object, engine]))
+  );
+}
+
+const obj1Match = expect.objectContaining({
+  pureName: 'obj1',
+});
+const view1Match = expect.objectContaining({
+  pureName: 'obj1',
+  columns: expect.arrayContaining([
+    expect.objectContaining({
+      columnName: 'id',
+    }),
+  ]),
+});
+
+describe('Object analyse', () => {
+  test.each(flatSource())(
+    'Full analysis - %s - %s',
+    testWrapper(async (conn, driver, type, object, engine) => {
+      for (const sql of initSql) await driver.query(conn, sql);
+
+      await driver.query(conn, object.create1);
+      const structure = await driver.analyseFull(conn);
+
+      expect(structure[type].length).toEqual(1);
+      expect(structure[type][0]).toEqual(type.includes('views') ? view1Match : obj1Match);
+    })
+  );
+
+  test.each(flatSource())(
+    'Incremental analysis - add - %s - %s',
+    testWrapper(async (conn, driver, type, object, engine) => {
+      for (const sql of initSql) await driver.query(conn, sql);
+
+      await driver.query(conn, object.create2);
+      const structure1 = await driver.analyseFull(conn);
+      await driver.query(conn, object.create1);
+      const structure2 = await driver.analyseIncremental(conn, structure1);
+
+      expect(structure2[type].length).toEqual(2);
+      expect(structure2[type].find(x => x.pureName == 'obj1')).toEqual(type.includes('views') ? view1Match : obj1Match);
+    })
+  );
+
+  test.each(flatSource())(
+    'Incremental analysis - drop - %s - %s',
+    testWrapper(async (conn, driver, type, object, engine) => {
+      for (const sql of initSql) await driver.query(conn, sql);
+
+      await driver.query(conn, object.create1);
+      await driver.query(conn, object.create2);
+      const structure1 = await driver.analyseFull(conn);
+      await driver.query(conn, object.drop2);
+      const structure2 = await driver.analyseIncremental(conn, structure1);
+
+      expect(structure2[type].length).toEqual(1);
+      expect(structure2[type][0]).toEqual(type.includes('views') ? view1Match : obj1Match);
+    })
+  );
+
+  test.each(flatSource())(
+    'Create SQL - add - %s - %s',
+    testWrapper(async (conn, driver, type, object, engine) => {
+      for (const sql of initSql) await driver.query(conn, sql);
+
+      await driver.query(conn, object.create1);
+      const structure1 = await driver.analyseFull(conn);
+      await driver.query(conn, object.drop1);
+      const structure2 = await driver.analyseIncremental(conn, structure1);
+      expect(structure2[type].length).toEqual(0);
+
+      await driver.query(conn, structure1[type][0].createSql);
+
+      const structure3 = await driver.analyseIncremental(conn, structure2);
+
+      expect(structure3[type].length).toEqual(1);
+      expect(structure3[type][0]).toEqual(type.includes('views') ? view1Match : obj1Match);
+    })
+  );
+});

integration-tests/__tests__/query.spec.js

@@ -0,0 +1,163 @@
+const engines = require('../engines');
+const { splitQuery } = require('dbgate-query-splitter');
+const { testWrapper } = require('../tools');
+
+const initSql = ['CREATE TABLE t1 (id int)', 'INSERT INTO t1 (id) VALUES (1)', 'INSERT INTO t1 (id) VALUES (2)'];
+
+expect.extend({
+  dataRow(row, expected) {
+    for (const key in expected) {
+      if (row[key] != expected[key]) {
+        return {
+          pass: false,
+          message: () => `Different key: ${key}`,
+        };
+      }
+    }
+    return {
+      pass: true,
+      message: () => '',
+    };
+  },
+});
+
+class StreamHandler {
+  constructor(resolve) {
+    this.results = [];
+    this.resolve = resolve;
+    this.infoRows = [];
+  }
+  row(row) {
+    this.results[this.results.length - 1].rows.push(row);
+  }
+  recordset(columns) {
+    this.results.push({
+      columns,
+      rows: [],
+    });
+  }
+  done(result) {
+    this.resolve(this.results);
+  }
+  info(msg) {
+    this.infoRows.push(msg);
+  }
+}
+
+function executeStreamItem(driver, conn, sql) {
+  return new Promise(resolve => {
+    const handler = new StreamHandler(resolve);
+    driver.stream(conn, sql, handler);
+  });
+}
+
+async function executeStream(driver, conn, sql) {
+  const results = [];
+  for (const sqlItem of splitQuery(sql, driver.getQuerySplitterOptions('stream'))) {
+    const item = await executeStreamItem(driver, conn, sqlItem);
+    results.push(...item);
+  }
+  return results;
+}
+
+describe('Query', () => {
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Simple query - %s',
+    testWrapper(async (conn, driver, engine) => {
+      for (const sql of initSql) await driver.query(conn, sql);
+
+      const res = await driver.query(conn, 'SELECT id FROM t1 ORDER BY id');
+      expect(res.columns).toEqual([
+        expect.objectContaining({
+          columnName: 'id',
+        }),
+      ]);
+
+      expect(res.rows).toEqual([
+        expect.dataRow({
+          id: 1,
+        }),
+        expect.dataRow({
+          id: 2,
+        }),
+      ]);
+    })
+  );
+
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Simple stream query - %s',
+    testWrapper(async (conn, driver, engine) => {
+      for (const sql of initSql) await driver.query(conn, sql);
+      const results = await executeStream(driver, conn, 'SELECT id FROM t1 ORDER BY id');
+      expect(results.length).toEqual(1);
+      const res = results[0];
+
+      expect(res.columns).toEqual([expect.objectContaining({ columnName: 'id' })]);
+      expect(res.rows).toEqual([expect.dataRow({ id: 1 }), expect.dataRow({ id: 2 })]);
+    })
+  );
+
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'More queries - %s',
+    testWrapper(async (conn, driver, engine) => {
+      for (const sql of initSql) await driver.query(conn, sql);
+      const results = await executeStream(
+        driver,
+        conn,
+        'SELECT id FROM t1 ORDER BY id; SELECT id FROM t1 ORDER BY id DESC'
+      );
+      expect(results.length).toEqual(2);
+
+      const res1 = results[0];
+      expect(res1.columns).toEqual([expect.objectContaining({ columnName: 'id' })]);
+      expect(res1.rows).toEqual([expect.dataRow({ id: 1 }), expect.dataRow({ id: 2 })]);
+
+      const res2 = results[1];
+      expect(res2.columns).toEqual([expect.objectContaining({ columnName: 'id' })]);
+      expect(res2.rows).toEqual([expect.dataRow({ id: 2 }), expect.dataRow({ id: 1 })]);
+    })
+  );
+
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Script - return data - %s',
+    testWrapper(async (conn, driver, engine) => {
+      const results = await executeStream(
+        driver,
+        conn,
+        'CREATE TABLE t1 (id int); INSERT INTO t1 (id) VALUES (1); INSERT INTO t1 (id) VALUES (2); SELECT id FROM t1 ORDER BY id; '
+      );
+      expect(results.length).toEqual(1);
+
+      const res1 = results[0];
+      expect(res1.columns).toEqual([expect.objectContaining({ columnName: 'id' })]);
+      expect(res1.rows).toEqual([expect.dataRow({ id: 1 }), expect.dataRow({ id: 2 })]);
+    })
+  );
+
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Script - no data - %s',
+    testWrapper(async (conn, driver, engine) => {
+      const results = await executeStream(
+        driver,
+        conn,
+        'CREATE TABLE t1 (id int); INSERT INTO t1 (id) VALUES (1); INSERT INTO t1 (id) VALUES (2) '
+      );
+      expect(results.length).toEqual(0);
+    })
+  );
+
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Save data query - %s',
+    testWrapper(async (conn, driver, engine) => {
+      for (const sql of initSql) await driver.query(conn, sql);
+
+      await driver.script(
+        conn,
+        'INSERT INTO t1 (id) VALUES (3);INSERT INTO t1 (id) VALUES (4);UPDATE t1 SET id=10 WHERE id=1;DELETE FROM t1 WHERE id=2;'
+      );
+      const res = await driver.query(conn, 'SELECT COUNT(*) AS cnt FROM t1');
+      // console.log(res);
+      expect(res.rows[0].cnt == 3).toBeTruthy();
+    })
+  );
+});

integration-tests/__tests__/table-analyse.spec.js

@@ -0,0 +1,164 @@
+const engines = require('../engines');
+const { testWrapper } = require('../tools');
+
+const t1Sql = 'CREATE TABLE t1 (id int not null primary key, val1 varchar(50) null)';
+const ix1Sql = 'CREATE index ix1 ON t1(val1, id)';
+const t2Sql = 'CREATE TABLE t2 (id int not null primary key, val2 varchar(50) null unique)';
+const t3Sql = 'CREATE TABLE t3 (id int not null primary key, valfk int, foreign key (valfk) references t2(id))';
+// const fkSql = 'ALTER TABLE t3 ADD FOREIGN KEY (valfk) REFERENCES t2(id)'
+
+const txMatch = (tname, vcolname, nextcol) =>
+  expect.objectContaining({
+    pureName: tname,
+    columns: [
+      expect.objectContaining({
+        columnName: 'id',
+        notNull: true,
+        dataType: expect.stringMatching(/int/i),
+      }),
+      expect.objectContaining({
+        columnName: vcolname,
+        notNull: false,
+        dataType: expect.stringMatching(/.*char.*\(50\)/),
+      }),
+      ...(nextcol
+        ? [
+            expect.objectContaining({
+              columnName: 'nextcol',
+              notNull: false,
+              dataType: expect.stringMatching(/.*char.*\(50\)/),
+            }),
+          ]
+        : []),
+    ],
+    primaryKey: expect.objectContaining({
+      columns: [
+        expect.objectContaining({
+          columnName: 'id',
+        }),
+      ],
+    }),
+  });
+
+const t1Match = txMatch('t1', 'val1');
+const t2Match = txMatch('t2', 'val2');
+const t2NextColMatch = txMatch('t2', 'val2', true);
+
+describe('Table analyse', () => {
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Table structure - full analysis - %s',
+    testWrapper(async (conn, driver, engine) => {
+      await driver.query(conn, t1Sql);
+
+      const structure = await driver.analyseFull(conn);
+
+      expect(structure.tables.length).toEqual(1);
+      expect(structure.tables[0]).toEqual(t1Match);
+    })
+  );
+
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Table add - incremental analysis - %s',
+    testWrapper(async (conn, driver, engine) => {
+      await driver.query(conn, t2Sql);
+
+      const structure1 = await driver.analyseFull(conn);
+      expect(structure1.tables.length).toEqual(1);
+      expect(structure1.tables[0]).toEqual(t2Match);
+
+      await driver.query(conn, t1Sql);
+      const structure2 = await driver.analyseIncremental(conn, structure1);
+
+      expect(structure2.tables.length).toEqual(2);
+      expect(structure2.tables.find(x => x.pureName == 't1')).toEqual(t1Match);
+      expect(structure2.tables.find(x => x.pureName == 't2')).toEqual(t2Match);
+    })
+  );
+
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Table remove - incremental analysis - %s',
+    testWrapper(async (conn, driver, engine) => {
+      await driver.query(conn, t1Sql);
+      await driver.query(conn, t2Sql);
+      const structure1 = await driver.analyseFull(conn);
+      expect(structure1.tables.length).toEqual(2);
+      expect(structure1.tables.find(x => x.pureName == 't1')).toEqual(t1Match);
+      expect(structure1.tables.find(x => x.pureName == 't2')).toEqual(t2Match);
+
+      await driver.query(conn, 'DROP TABLE t2');
+      const structure2 = await driver.analyseIncremental(conn, structure1);
+
+      expect(structure2.tables.length).toEqual(1);
+      expect(structure2.tables[0]).toEqual(t1Match);
+    })
+  );
+
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Table change - incremental analysis - %s',
+    testWrapper(async (conn, driver, engine) => {
+      await driver.query(conn, t1Sql);
+      await driver.query(conn, t2Sql);
+      const structure1 = await driver.analyseFull(conn);
+
+      if (engine.dbSnapshotBySeconds) await new Promise(resolve => setTimeout(resolve, 1100));
+
+      await driver.query(conn, 'ALTER TABLE t2 ADD nextcol varchar(50)');
+      const structure2 = await driver.analyseIncremental(conn, structure1);
+
+      expect(structure2).toBeTruthy(); // if falsy, no modification is detected
+
+      expect(structure2.tables.length).toEqual(2);
+      expect(structure2.tables.find(x => x.pureName == 't1')).toEqual(t1Match);
+      expect(structure2.tables.find(x => x.pureName == 't2')).toEqual(t2NextColMatch);
+    })
+  );
+
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Index - full analysis - %s',
+    testWrapper(async (conn, driver, engine) => {
+      await driver.query(conn, t1Sql);
+      await driver.query(conn, ix1Sql);
+      const structure = await driver.analyseFull(conn);
+
+      const t1 = structure.tables.find(x => x.pureName == 't1');
+      expect(t1.indexes.length).toEqual(1);
+      expect(t1.indexes[0].columns.length).toEqual(2);
+      expect(t1.indexes[0].columns[0]).toEqual(expect.objectContaining({ columnName: 'val1' }));
+      expect(t1.indexes[0].columns[1]).toEqual(expect.objectContaining({ columnName: 'id' }));
+    })
+  );
+
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Unique - full analysis - %s',
+    testWrapper(async (conn, driver, engine) => {
+      await driver.query(conn, t2Sql);
+      const structure = await driver.analyseFull(conn);
+
+      const t2 = structure.tables.find(x => x.pureName == 't2');
+      // const indexesAndUniques = [...t2.uniques, ...t2.indexes];
+      expect(t2.uniques.length).toEqual(1);
+      expect(t2.uniques[0].columns.length).toEqual(1);
+      expect(t2.uniques[0].columns[0]).toEqual(expect.objectContaining({ columnName: 'val2' }));
+    })
+  );
+
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Foreign key - full analysis - %s',
+    testWrapper(async (conn, driver, engine) => {
+      await driver.query(conn, t2Sql);
+      await driver.query(conn, t3Sql);
+      // await driver.query(conn, fkSql);
+
+      const structure = await driver.analyseFull(conn);
+
+      const t3 = structure.tables.find(x => x.pureName == 't3');
+      console.log('T3', t3.foreignKeys[0].columns);
+      expect(t3.foreignKeys.length).toEqual(1);
+      expect(t3.foreignKeys[0].columns.length).toEqual(1);
+      expect(t3.foreignKeys[0]).toEqual(expect.objectContaining({ refTableName: 't2' }));
+      expect(t3.foreignKeys[0].columns[0]).toEqual(
+        expect.objectContaining({ columnName: 'valfk', refColumnName: 'id' })
+      );
+    })
+  );
+});

integration-tests/__tests__/table-create.spec.js

@@ -0,0 +1,153 @@
+const _ = require('lodash');
+const fp = require('lodash/fp');
+const engines = require('../engines');
+const { testWrapper } = require('../tools');
+const { extendDatabaseInfo } = require('dbgate-tools');
+
+function createExpector(value) {
+  return _.cloneDeepWith(value, x => {
+    if (_.isPlainObject(x)) {
+      return expect.objectContaining(_.mapValues(x, y => createExpector(y)));
+    }
+  });
+}
+
+function omitTableSpecificInfo(table) {
+  return {
+    ...table,
+    columns: table.columns.map(fp.omit(['dataType'])),
+  };
+}
+
+function checkTableStructure2(t1, t2) {
+  // expect(t1.pureName).toEqual(t2.pureName)
+  expect(t2).toEqual(createExpector(omitTableSpecificInfo(t1)));
+}
+
+async function testTableCreate(conn, driver, table) {
+  await driver.query(conn, `create table t0 (id int not null primary key)`);
+
+  const dmp = driver.createDumper();
+  const table1 = {
+    ...table,
+    pureName: 'tested',
+  };
+  dmp.createTable(table1);
+
+  console.log('RUNNING CREATE SQL', driver.engine, ':', dmp.s);
+  await driver.script(conn, dmp.s);
+
+  const db = extendDatabaseInfo(await driver.analyseFull(conn));
+  const table2 = db.tables.find(x => x.pureName == 'tested');
+
+  checkTableStructure2(table1, table2);
+}
+
+describe('Table create', () => {
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Simple table - %s',
+    testWrapper(async (conn, driver, engine) => {
+      await testTableCreate(conn, driver, {
+        columns: [
+          {
+            columnName: 'col1',
+            dataType: 'int',
+            notNull: true,
+          },
+        ],
+        primaryKey: {
+          columns: [{ columnName: 'col1' }],
+        },
+      });
+    })
+  );
+
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Table with index - %s',
+    testWrapper(async (conn, driver, engine) => {
+      await testTableCreate(conn, driver, {
+        columns: [
+          {
+            columnName: 'col1',
+            dataType: 'int',
+            notNull: true,
+          },
+          {
+            columnName: 'col2',
+            dataType: 'int',
+            notNull: true,
+          },
+        ],
+        primaryKey: {
+          columns: [{ columnName: 'col1' }],
+        },
+        indexes: [
+          {
+            constraintName: 'ix1',
+            pureName: 'tested',
+            columns: [{ columnName: 'col2' }],
+          },
+        ],
+      });
+    })
+  );
+
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Table with foreign key - %s',
+    testWrapper(async (conn, driver, engine) => {
+      await testTableCreate(conn, driver, {
+        columns: [
+          {
+            columnName: 'col1',
+            dataType: 'int',
+            notNull: true,
+          },
+          {
+            columnName: 'col2',
+            dataType: 'int',
+            notNull: true,
+          },
+        ],
+        primaryKey: {
+          columns: [{ columnName: 'col1' }],
+        },
+        foreignKeys: [
+          {
+            pureName: 'tested',
+            refTableName: 't0',
+            columns: [{ columnName: 'col2', refColumnName: 'id' }],
+          },
+        ],
+      });
+    })
+  );
+
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Table with unique - %s',
+    testWrapper(async (conn, driver, engine) => {
+      await testTableCreate(conn, driver, {
+        columns: [
+          {
+            columnName: 'col1',
+            dataType: 'int',
+            notNull: true,
+          },
+          {
+            columnName: 'col2',
+            dataType: 'int',
+            notNull: true,
+          },
+        ],
+        primaryKey: {
+          columns: [{ columnName: 'col1' }],
+        },
+        uniques: [
+          {
+            pureName: 'tested',
+            columns: [{ columnName: 'col2' }],
+          },
+        ],
+      });
+    })
+  );
+});

integration-tests/docker-compose.yaml

@@ -0,0 +1,64 @@
+version: '3'
+services:
+  # postgres:
+  #   image: postgres
+  #   restart: always
+  #   environment:
+  #     POSTGRES_PASSWORD: Pwd2020Db
+  #   ports: 
+  #     - 15000:5432
+
+  # mariadb:
+  #   image: mariadb
+  #   command: --default-authentication-plugin=mysql_native_password
+  #   restart: always
+  #   ports: 
+  #     - 15004:3306
+  #   environment:
+  #     - MYSQL_ROOT_PASSWORD=Pwd2020Db
+
+  # mysql:
+  #   image: mysql:8.0.18
+  #   command: --default-authentication-plugin=mysql_native_password
+  #   restart: always
+  #   ports: 
+  #     - 15001:3306
+  #   environment:
+  #     - MYSQL_ROOT_PASSWORD=Pwd2020Db
+
+  mssql:
+    image: mcr.microsoft.com/mssql/server
+    restart: always
+    ports: 
+      - 15002:1433
+    environment:
+      - ACCEPT_EULA=Y
+      - SA_PASSWORD=Pwd2020Db
+      - MSSQL_PID=Express
+      
+  # cockroachdb:
+  #   image: cockroachdb/cockroach
+  #   ports:
+  #     - 15003:26257
+  #   command: start-single-node --insecure
+
+#   mongodb:
+#     image: mongo:4.0.12
+#     restart: always
+#     volumes:
+#       - mongo-data:/data/db
+#       - mongo-config:/data/configdb
+#     ports:
+#       - 27017:27017
+
+
+#   cockroachdb-init:
+#     image: cockroachdb/cockroach
+#     # build: cockroach
+#     # entrypoint: /cockroach/init.sh 
+#     entrypoint: ./cockroach sql --insecure --host="cockroachdb" --execute="CREATE DATABASE IF NOT EXISTS test;"
+
+#     depends_on:
+#       - cockroachdb
+#     restart: on-failure
+

integration-tests/engines.js

@@ -0,0 +1,150 @@
+const views = {
+  type: 'views',
+  create1: 'CREATE VIEW obj1 AS SELECT id FROM t1',
+  create2: 'CREATE VIEW obj2 AS SELECT id FROM t2',
+  drop1: 'DROP VIEW obj1',
+  drop2: 'DROP VIEW obj2',
+};
+const matviews = {
+  type: 'matviews',
+  create1: 'CREATE MATERIALIZED VIEW obj1 AS SELECT id FROM t1',
+  create2: 'CREATE MATERIALIZED VIEW obj2 AS SELECT id FROM t2',
+  drop1: 'DROP MATERIALIZED VIEW obj1',
+  drop2: 'DROP MATERIALIZED VIEW obj2',
+};
+
+const engines = [
+  {
+    label: 'MySQL',
+    connection: {
+      engine: 'mysql@dbgate-plugin-mysql',
+      password: 'Pwd2020Db',
+      user: 'root',
+      server: 'mysql',
+      port: 3306,
+    },
+    local: {
+      server: 'localhost',
+      port: 15001,
+    },
+    // skipOnCI: true,
+    objects: [views],
+    dbSnapshotBySeconds: true,
+  },
+  {
+    label: 'MariaDB',
+    connection: {
+      engine: 'mariadb@dbgate-plugin-mysql',
+      password: 'Pwd2020Db',
+      user: 'root',
+      server: 'mysql',
+      port: 3306,
+    },
+    local: {
+      server: 'localhost',
+      port: 15004,
+    },
+    skipOnCI: true,
+    objects: [views],
+    dbSnapshotBySeconds: true,
+  },
+  {
+    label: 'PostgreSQL',
+    connection: {
+      engine: 'postgres@dbgate-plugin-postgres',
+      password: 'Pwd2020Db',
+      user: 'postgres',
+      server: 'postgres',
+      port: 5432,
+    },
+    local: {
+      server: 'localhost',
+      port: 15000,
+    },
+    objects: [
+      views,
+      matviews,
+      {
+        type: 'procedures',
+        create1: 'CREATE PROCEDURE obj1() LANGUAGE SQL AS $$  select * from t1 $$',
+        create2: 'CREATE PROCEDURE obj2() LANGUAGE SQL AS $$  select * from t2 $$',
+        drop1: 'DROP PROCEDURE obj1',
+        drop2: 'DROP PROCEDURE obj2',
+      },
+      {
+        type: 'functions',
+        create1:
+          'CREATE FUNCTION obj1() returns int LANGUAGE plpgsql AS $$ declare  res integer; begin select count(*) into res from t1; return res; end; $$',
+        create2:
+          'CREATE FUNCTION obj2() returns int LANGUAGE plpgsql AS $$ declare res integer; begin select count(*) into res from t2; return res; end; $$',
+        drop1: 'DROP FUNCTION obj1',
+        drop2: 'DROP FUNCTION obj2',
+      },
+    ],
+  },
+  {
+    label: 'SQL Server',
+    connection: {
+      engine: 'mssql@dbgate-plugin-mssql',
+      password: 'Pwd2020Db',
+      user: 'sa',
+      server: 'mssql',
+      port: 1433,
+    },
+    local: {
+      server: 'localhost',
+      port: 15002,
+    },
+    objects: [
+      views,
+      {
+        type: 'procedures',
+        create1: 'CREATE PROCEDURE obj1 AS SELECT id FROM t1',
+        create2: 'CREATE PROCEDURE obj2 AS SELECT id FROM t2',
+        drop1: 'DROP PROCEDURE obj1',
+        drop2: 'DROP PROCEDURE obj2',
+      },
+    ],
+  },
+  {
+    label: 'SQLite',
+    generateDbFile: true,
+    connection: {
+      engine: 'sqlite@dbgate-plugin-sqlite',
+    },
+    objects: [views],
+  },
+  {
+    label: 'CockroachDB',
+    connection: {
+      engine: 'cockroach@dbgate-plugin-postgres',
+      user: 'root',
+      server: 'cockroachdb',
+      port: 26257,
+    },
+    local: {
+      server: 'localhost',
+      port: 15003,
+    },
+    skipOnCI: true,
+    objects: [views, matviews],
+  },
+];
+
+const filterLocal = [
+  // filter local testing
+  '-MySQL',
+  '-MariaDB',
+  '-PostgreSQL',
+  '-SQL Server',
+  'SQLite',
+  '-CockroachDB',
+];
+
+const enginesPostgre = engines.filter(x => x.label == 'PostgreSQL');
+
+module.exports = process.env.CITEST
+  ? engines.filter(x => !x.skipOnCI)
+  : engines.filter(x => filterLocal.find(y => x.label == y));
+
+module.exports.enginesPostgre = enginesPostgre;

integration-tests/package.json

@@ -0,0 +1,30 @@
+{
+  "name": "dbgate-integration-tests",
+  "version": "5.0.0-alpha.1",
+  "homepage": "https://dbgate.org/",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/dbgate/dbgate.git"
+  },
+  "author": "Jan Prochazka",
+  "license": "MIT",
+  "scripts": {
+    "wait:local": "cross-env DEVMODE=1 LOCALTEST=1 node wait.js",
+    "wait:ci": "cross-env DEVMODE=1 CITEST=1 node wait.js",
+
+    "test:local": "cross-env DEVMODE=1 LOCALTEST=1 jest",
+    "test:local:path": "cross-env DEVMODE=1 LOCALTEST=1 jest --runTestsByPath __tests__/data-duplicator.spec.js",
+    
+    "test:ci": "cross-env DEVMODE=1 CITEST=1 jest --runInBand --json --outputFile=result.json --testLocationInResults",
+    
+    "run:local": "docker-compose down && docker-compose up -d && yarn wait:local && yarn test:local"
+  },
+  "jest": {
+    "testTimeout": 5000
+  },
+  "devDependencies": {
+    "cross-env": "^7.0.3",
+    "jest": "^27.0.1"
+  },
+  "dependencies": {}
+}

integration-tests/tools.js

@@ -0,0 +1,63 @@
+global.DBGATE_TOOLS = require('dbgate-tools');
+const requireEngineDriver = require('dbgate-api/src/utility/requireEngineDriver');
+const crypto = require('crypto');
+
+function randomDbName() {
+  const generatedKey = crypto.randomBytes(6);
+  const newKey = generatedKey.toString('hex');
+  return `db${newKey}`;
+}
+
+function extractConnection(engine) {
+  const { connection } = engine;
+
+  if (process.env.LOCALTEST && engine.local) {
+    return {
+      ...connection,
+      ...engine.local,
+    };
+  }
+
+  return connection;
+}
+
+async function connect(engine, database) {
+  const connection = extractConnection(engine);
+  const driver = requireEngineDriver(connection);
+
+  if (engine.generateDbFile) {
+    const conn = await driver.connect({
+      ...connection,
+      databaseFile: `dbtemp/${database}`,
+    });
+    return conn;
+  } else {
+    const conn = await driver.connect(connection);
+    await driver.query(conn, `CREATE DATABASE ${database}`);
+    await driver.close(conn);
+
+    const res = await driver.connect({
+      ...connection,
+      database,
+    });
+    return res;
+  }
+}
+
+const testWrapper = body => async (label, ...other) => {
+  const engine = other[other.length - 1];
+  const driver = requireEngineDriver(engine.connection);
+  const conn = await connect(engine, randomDbName());
+  try {
+    await body(conn, driver, ...other);
+  } finally {
+    await driver.close(conn);
+  }
+};
+
+module.exports = {
+  randomDbName,
+  connect,
+  extractConnection,
+  testWrapper,
+};

integration-tests/wait.js

@@ -0,0 +1,29 @@
+const requireEngineDriver = require('dbgate-api/src/utility/requireEngineDriver');
+const engines = require('./engines');
+const { extractConnection } = require('./tools');
+global.DBGATE_TOOLS = require('dbgate-tools');
+
+async function connectEngine(engine) {
+  const connection = extractConnection(engine);
+  const driver = requireEngineDriver(connection);
+  for (;;) {
+    try {
+      const conn = await driver.connect(connection);
+      await driver.getVersion(conn);
+      console.log(`Connect to ${engine.label} - OK`);
+      await driver.close(conn);
+      return;
+    } catch (err) {
+      console.log(`Waiting for ${engine.label}, error: ${err.message}`);
+      await new Promise(resolve => setTimeout(resolve, 2500));
+      continue;
+    }
+  }
+}
+
+async function run() {
+  await new Promise(resolve => setTimeout(resolve, 10000));
+  await Promise.all(engines.map(engine => connectEngine(engine)));
+}
+
+run();

misc/convert-icons.sh

@@ -0,0 +1,109 @@
+# magick icon.svg -resize 1024x1024 -transparent white -background transparent app/icon1024.png
+
+# magick app/icon1024.png -resize 512x512 app/icon512.png
+# magick app/icon1024.png -resize 256x256 app/icon.png
+# magick app/icon1024.png -resize 32x32 app/icon32.png
+# magick icon.svg -define icon:auto-resize="256,128,96,64,48,32,16" -transparent white -background transparent app/icon.ico
+
+# # magick icon.svg -resize 512x512 -transparent white -background transparent app/icon512.png
+# # magick icon.svg -resize 256x256 -transparent white -background transparent app/icon.png
+# # magick icon.svg -resize 32x32 -transparent white -background transparent app/icon32.png
+# # magick icon.svg -define icon:auto-resize="256,128,96,64,48,32,16" -transparent white -background transparent app/icon.ico
+
+# magick app/icon1024.png -resize 512x512 app/icons/512x512.png
+# magick app/icon1024.png -resize 256x256 app/icons/256x256.png
+# magick app/icon1024.png -resize 128x128 app/icons/128x128.png
+# magick app/icon1024.png -resize 64x64 app/icons/64x64.png
+# magick app/icon1024.png -resize 48x48 app/icons/48x48.png
+# magick app/icon1024.png -resize 32x32 app/icons/32x32.png
+# magick app/icon1024.png -resize 16x16 app/icons/16x16.png
+
+# # magick icon.svg -resize 16x16 -transparent white -background transparent app/icons/16x16.png
+# # magick icon.svg -resize 32x32 -transparent white -background transparent app/icons/32x32.png
+# # magick icon.svg -resize 48x48 -transparent white -background transparent app/icons/48x48.png
+# # magick icon.svg -resize 64x64 -transparent white -background transparent app/icons/64x64.png
+# # magick icon.svg -resize 128x128 -transparent white -background transparent app/icons/128x128.png
+# # magick icon.svg -resize 256x256 -transparent white -background transparent app/icons/256x256.png
+# # magick icon.svg -resize 512x512 -transparent white -background transparent app/icons/512x512.png
+
+# magick icon.svg -resize 1024x1024 icon.png
+# magick icon.svg -resize 1024x1024 -transparent white -background transparent icon.png
+
+STROKE_WIDTH=30
+LEFT=150
+RIGHT=850
+
+
+
+magick  \
+    \( \
+        -size 1000x1000 -define gradient:direction=east 'gradient:#0050b3-#1890ff' \
+        \( +clone -fill Black -colorize 100 \
+        -fill White -stroke White -draw "arc $LEFT,750 $RIGHT,950 0,360" -draw "rectangle $LEFT,150 $RIGHT,850" \
+        \) \
+        -alpha off \
+        -compose CopyOpacity -composite \
+    \) \
+    \( \
+        -size 1000x1000 -define gradient:direction=east 'gradient:#096dd9-#40a9ff' \
+        \( +clone -fill Black -colorize 100 \
+        -fill White -draw "arc $LEFT,50 $RIGHT,250 0,360" \
+        \) \
+        -alpha off \
+        -compose CopyOpacity -composite \
+    \) \
+    -compose Over -composite \
+    -strokewidth $STROKE_WIDTH -stroke '#0050b3' -fill transparent \
+    -draw "arc $LEFT,225 $RIGHT,425  0,180" \
+    -draw "arc $LEFT,400 $RIGHT,600  0,180" \
+    -draw "arc $LEFT,575 $RIGHT,775  0,180" \
+    -draw "arc $LEFT,750 $RIGHT,950  0,180" \
+    -draw "arc $LEFT,50 $RIGHT,250  0,360" \
+    -draw "line $LEFT,150 $LEFT,850" \
+    -draw "line $RIGHT,150 $RIGHT,850" \
+    -fill '#fafafa' -stroke '#8c8c8c' -strokewidth 3 \
+    -pointsize 800 -font './Mcbungus-Regular.ttf' \
+    -gravity center \
+    -draw 'text 0,100 "G"' \
+    icon.png
+
+
+# magick \
+#   \( \
+#     -size 300x300 gradient:red-blue \
+#     \( +clone -fill Black -colorize 100 \
+#       -fill White -draw "polygon 50,50 250,50 200,200" \
+#     \) \
+#     -alpha off \
+#     -compose CopyOpacity -composite \
+#   \) \
+#   \( \
+#     -size 300x300 'gradient:#f80-#08f' \
+#     \( +clone -fill Black -colorize 100 \
+#       -fill White -draw "polygon 50,150 250,150 200,300" \
+#     \) \
+#     -alpha off \
+#     -compose CopyOpacity -composite \
+#   \) \
+#   -compose Over -composite \
+#   icon.png    
+
+magick icon.png -resize 512x512! ../app/icon512.png
+magick icon.png -resize 256x256! ../app/icon.png
+magick icon.png -resize 32x32! ../app/icon32.png
+magick icon.png -define icon:auto-resize="256,128,96,64,48,32,16" ../app/icon.ico
+
+magick icon.png -resize 512x512! ../app/icons/512x512.png
+magick icon.png -resize 256x256! ../app/icons/256x256.png
+magick icon.png -resize 128x128! ../app/icons/128x128.png
+magick icon.png -resize 64x64! ../app/icons/64x64.png
+magick icon.png -resize 48x48! ../app/icons/48x48.png
+magick icon.png -resize 32x32! ../app/icons/32x32.png
+magick icon.png -resize 16x16! ../app/icons/16x16.png
+
+magick icon.png -resize 192x192! ../packages/web/public/logo192.png
+magick icon.png -resize 512x512! ../packages/web/public/logo512.png
+magick icon.png -define icon:auto-resize="256,128,96,64,48,32,16" ../packages/web/public/favicon.ico
+
+convert icon.png -resize 800x800 -background transparent -gravity center -extent 1000x1000 iconmac.png
+magick composite  iconmac.png macbg.png -resize 600x600! ../app/icon512-mac.png

misc/play-dark-mode.svg

@@ -0,0 +1,14 @@
+<?xml version="1.0" encoding="iso-8859-1"?>
+<!-- Generator: Adobe Illustrator 18.1.1, SVG Export Plug-In . SVG Version: 6.00 Build 0)  -->
+<svg version="1.1" id="Capa_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px"
+	 viewBox="0 0 17.804 17.804" style="enable-background:new 0 0 17.804 17.804;" xml:space="preserve">
+<g>
+	<g id="c98_play">
+		<path fill='#ccc' d="M2.067,0.043C2.21-0.028,2.372-0.008,2.493,0.085l13.312,8.503c0.094,0.078,0.154,0.191,0.154,0.313
+			c0,0.12-0.061,0.237-0.154,0.314L2.492,17.717c-0.07,0.057-0.162,0.087-0.25,0.087l-0.176-0.04
+			c-0.136-0.065-0.222-0.207-0.222-0.361V0.402C1.844,0.25,1.93,0.107,2.067,0.043z"/>
+	</g>
+	<g id="Capa_1_78_">
+	</g>
+</g>
+</svg>

misc/play-light-mode.svg

@@ -0,0 +1,14 @@
+<?xml version="1.0" encoding="iso-8859-1"?>
+<!-- Generator: Adobe Illustrator 18.1.1, SVG Export Plug-In . SVG Version: 6.00 Build 0)  -->
+<svg version="1.1" id="Capa_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px"
+	 viewBox="0 0 17.804 17.804" style="enable-background:new 0 0 17.804 17.804;" xml:space="preserve">
+<g>
+	<g id="c98_play">
+		<path fill='#444' d="M2.067,0.043C2.21-0.028,2.372-0.008,2.493,0.085l13.312,8.503c0.094,0.078,0.154,0.191,0.154,0.313
+			c0,0.12-0.061,0.237-0.154,0.314L2.492,17.717c-0.07,0.057-0.162,0.087-0.25,0.087l-0.176-0.04
+			c-0.136-0.065-0.222-0.207-0.222-0.361V0.402C1.844,0.25,1.93,0.107,2.067,0.043z"/>
+	</g>
+	<g id="Capa_1_78_">
+	</g>
+</g>
+</svg>

package.json

@@ -1,15 +1,26 @@
 {
   "private": true,
-  "version": "3.9.6",
+  "version": "5.2.7",
   "name": "dbgate-all",
   "workspaces": [
-    "packages/*"
+    "packages/*",
+    "plugins/*",
+    "integration-tests"
   ],
   "scripts": {
-    "start:api": "yarn workspace dbgate-api start",
-    "start:api:portal": "yarn workspace dbgate-api start:portal",
-    "start:api:covid": "yarn workspace dbgate-api start:covid",
-    "start:web": "yarn workspace dbgate-web start",
+    "start:api": "yarn workspace dbgate-api start | pino-pretty",
+    "start:api:json": "yarn workspace dbgate-api start",
+    "start:app": "cd app && yarn start | pino-pretty",
+    "start:app:singledb": "CONNECTIONS=con1 SERVER_con1=localhost ENGINE_con1=mysql@dbgate-plugin-mysql USER_con1=root PASSWORD_con1=Pwd2020Db SINGLE_CONNECTION=con1 SINGLE_DATABASE=Chinook yarn start:app",
+    "start:api:debug": "cross-env DEBUG=* yarn workspace dbgate-api start",
+    "start:app:debug": "cd app && cross-env DEBUG=* yarn start",
+    "start:api:debug:ssh": "cross-env DEBUG=ssh yarn workspace dbgate-api start",
+    "start:app:debug:ssh": "cd app && cross-env DEBUG=ssh yarn start",
+    "start:api:portal": "yarn workspace dbgate-api start:portal | pino-pretty",
+    "start:api:singledb": "yarn workspace dbgate-api start:singledb | pino-pretty",
+    "start:api:auth": "yarn workspace dbgate-api start:auth | pino-pretty",
+    "start:api:dblogin": "yarn workspace dbgate-api start:dblogin | pino-pretty",
+    "start:web": "yarn workspace dbgate-web dev",
     "start:sqltree": "yarn workspace dbgate-sqltree start",
     "start:tools": "yarn workspace dbgate-tools start",
     "start:datalib": "yarn workspace dbgate-datalib start",
@@ -18,34 +29,42 @@
     "build:datalib": "yarn workspace dbgate-datalib build",
     "build:filterparser": "yarn workspace dbgate-filterparser build",
     "build:tools": "yarn workspace dbgate-tools build",
-    "build:lib": "yarn build:tools && yarn build:sqltree && yarn build:filterparser && yarn build:datalib",
-    "build:app": "cd app && yarn install && yarn build",
+    "build:lib": "yarn build:sqltree && yarn build:tools && yarn build:filterparser && yarn build:datalib",
+    "build:app": "yarn plugins:copydist && cd app && yarn install && yarn build",
     "build:api": "yarn workspace dbgate-api build",
-    "build:web:docker": "yarn workspace dbgate-web build:docker",
-    "build:app:local": "cd app && yarn build:local",
+    "build:web:docker": "yarn workspace dbgate-web build",
+    "build:plugins:frontend": "workspaces-run --only=\"dbgate-plugin-*\" -- yarn build:frontend",
+    "build:plugins:frontend:watch": "workspaces-run --parallel --only=\"dbgate-plugin-*\" -- yarn build:frontend:watch",
+    "plugins:copydist": "workspaces-run --only=\"dbgate-plugin-*\" -- yarn copydist",
+    "build:app:local": "yarn plugins:copydist && cd app && yarn build:local",
     "start:app:local": "cd app && yarn start:local",
     "setCurrentVersion": "node setCurrentVersion",
     "generatePadFile": "node generatePadFile",
+    "adjustPackageJson": "node adjustPackageJson",
     "fillNativeModules": "node fillNativeModules",
-    "fillNativeModulesElectron": "node fillNativeModules --eletron",
+    "fillNativeModulesElectron": "node fillNativeModules --electron",
+    "fillPackagedPlugins": "node fillPackagedPlugins",
+    "resetPackagedPlugins": "node resetPackagedPlugins",
     "prettier": "prettier --write packages/api/src && prettier --write packages/datalib/src && prettier --write packages/filterparser/src && prettier --write packages/sqltree/src && prettier --write packages/tools/src && prettier --write packages/types && prettier --write packages/web/src && prettier --write app/src",
-    "copy:docker:build": "copyfiles packages/api/dist/* docker -f && copyfiles packages/web/build/* docker -u 2 && copyfiles \"packages/web/build/**/*\" docker -u 2",
-    "prepare:docker": "yarn build:web:docker && yarn build:api && yarn copy:docker:build",
-    "prepare": "yarn build:lib",
+    "copy:docker:build": "copyfiles packages/api/dist/* docker -f && copyfiles packages/web/public/* docker -u 2 && copyfiles \"packages/web/public/**/*\" docker -u 2 && copyfiles \"plugins/dist/**/*\" docker/plugins -u 2",
+    "install:sqlite:docker": "cd docker && yarn init --yes && yarn add better-sqlite3 && cd ..",
+    "prepare:docker": "yarn plugins:copydist && yarn build:web:docker && yarn build:api && yarn copy:docker:build && yarn install:sqlite:docker",
     "start": "concurrently --kill-others-on-fail \"yarn start:api\" \"yarn start:web\"",
-    "lib": "concurrently --kill-others-on-fail \"yarn start:sqltree\" \"yarn start:filterparser\" \"yarn start:datalib\" \"yarn start:tools\"",
+    "lib": "concurrently --kill-others-on-fail \"yarn start:sqltree\" \"yarn start:filterparser\" \"yarn start:datalib\" \"yarn start:tools\" \"yarn build:plugins:frontend:watch\"",
     "ts:api": "yarn workspace dbgate-api ts",
     "ts:web": "yarn workspace dbgate-web ts",
     "ts": "yarn ts:api && yarn ts:web",
-    "postinstall": "patch-package && yarn fillNativeModules"
+    "postinstall": "yarn resetPackagedPlugins && yarn build:lib && patch-package && yarn fillNativeModules && yarn build:plugins:frontend",
+    "dbgate-serve": "node packages/dbgate/bin/dbgate-serve.js"
   },
   "dependencies": {
     "concurrently": "^5.1.0",
     "patch-package": "^6.2.1",
-    "socket.io": "^2.3.0"
+    "pino-pretty": "^9.1.1"
   },
   "devDependencies": {
     "copyfiles": "^2.2.0",
-    "prettier": "^2.2.1"
+    "prettier": "^2.2.1",
+    "workspaces-run": "^1.0.1"
   }
 }

packages/api/.covid-env

@@ -1,12 +0,0 @@
-CONNECTIONS=mysql
-
-LABEL_mysql=MySql
-SERVER_mysql=dbgate.org
-USER_mysql=reader
-PASSWORD_mysql=CovidReader2020
-PORT_mysql=3326
-ENGINE_mysql=mysql@dbgate-plugin-mysql
-SINGLE_CONNECTION=mysql
-SINGLE_DATABASE=covid
-
-PERMISSIONS=files/charts/read

packages/api/.env

@@ -1,23 +1,17 @@
-CONNECTIONS=mysql,postgres
+DEVMODE=1
+SHELL_SCRIPTING=1
 
-LABEL_mysql=MySql localhost
-SERVER_mysql=localhost
-USER_mysql=root
-PASSWORD_mysql=test
-PORT_mysql=3307
-ENGINE_mysql=mysql
+# PERMISSIONS=~widgets/app,~widgets/plugins
+# DISABLE_SHELL=1
+# HIDE_APP_EDITOR=1
 
-LABEL_postgres=Postgres localhost
-SERVER_postgres=localhost
-USER_postgres=postgres
-PASSWORD_postgres=test
-PORT_postgres=5433
-ENGINE_postgres=postgres
 
-TOOLBAR=home
-ICON_home=mdi mdi-home
-TITLE_home=Home
-PAGE_home=home.html
-STARTUP_PAGES=home
+# DEVWEB=1
+# LOGINS=admin,test
 
-PAGES_DIRECTORY=/home/jena/jenasoft/dbgate-web/pages
+# LOGIN_PASSWORD_admin=admin
+# LOGIN_PERMISSIONS_admin=*
+
+# LOGIN_PASSWORD_test=test
+# LOGIN_PERMISSIONS_test=~*, widgets/database
+# WORKSPACE_DIR=/home/jena/dbgate-data-2

packages/api/env/auth/.gitignore

@@ -0,0 +1 @@
+.env

packages/api/env/dblogin/.env

@@ -0,0 +1,14 @@
+DEVMODE=1
+
+CONNECTIONS=mysql
+SINGLE_CONNECTION=mysql
+# SINGLE_DATABASE=Chinook
+
+LABEL_mysql=MySql localhost
+SERVER_mysql=localhost
+# USER_mysql=root
+PORT_mysql=3306
+# PASSWORD_mysql=Pwd2020Db
+ENGINE_mysql=mysql@dbgate-plugin-mysql
+# PASSWORD_MODE_mysql=askPassword
+PASSWORD_MODE_mysql=askUser

packages/api/env/portal/.env

@@ -0,0 +1,62 @@
+DEVMODE=1
+
+CONNECTIONS=mysql,postgres,mongo,mongo2,mysqlssh,sqlite,relational
+
+LABEL_mysql=MySql localhost
+SERVER_mysql=localhost
+USER_mysql=root
+PASSWORD_mysql=test
+PORT_mysql=3307
+ENGINE_mysql=mysql@dbgate-plugin-mysql
+
+LABEL_postgres=Postgres localhost
+SERVER_postgres=localhost
+USER_postgres=postgres
+PASSWORD_postgres=test
+PORT_postgres=5433
+ENGINE_postgres=postgres@dbgate-plugin-postgres
+
+LABEL_mongo=Mongo URL
+URL_mongo=mongodb://localhost:27017
+ENGINE_mongo=mongo@dbgate-plugin-mongo
+
+LABEL_mongo2=Mongo Server
+SERVER_mongo2=localhost
+ENGINE_mongo2=mongo@dbgate-plugin-mongo
+
+LABEL_mysqlssh=MySql SSH
+SERVER_mysqlssh=localhost
+USER_mysqlssh=root
+PASSWORD_mysqlssh=xxx
+PORT_mysqlssh=3316
+ENGINE_mysqlssh=mysql@dbgate-plugin-mysql
+USE_SSH_mysqlssh=1
+SSH_HOST_mysqlssh=demo.dbgate.org
+SSH_PORT_mysqlssh=22
+SSH_MODE_mysqlssh=userPassword
+SSH_LOGIN_mysqlssh=root
+SSH_PASSWORD_mysqlssh=xxx
+
+LABEL_sqlite=sqlite
+FILE_sqlite=/home/jena/.dbgate/files/sqlite/feeds.sqlite
+ENGINE_sqlite=sqlite@dbgate-plugin-sqlite
+
+LABEL_relational=Relational dataset repo
+SERVER_relational=relational.fit.cvut.cz
+USER_relational=guest
+PASSWORD_relational=relational
+ENGINE_relational=mariadb@dbgate-plugin-mysql
+READONLY_relational=1
+
+# SETTINGS_dataGrid.showHintColumns=1
+
+# docker run -p 3000:3000 -e CONNECTIONS=mongo -e URL_mongo=mongodb://localhost:27017 -e ENGINE_mongo=mongo@dbgate-plugin-mongo -e LABEL_mongo=mongo dbgate/dbgate:beta
+
+# LOGINS=x,y
+# LOGIN_PASSWORD_x=x
+# LOGIN_PASSWORD_y=LOGIN_PASSWORD_y
+# LOGIN_PERMISSIONS_x=~*
+# LOGIN_PERMISSIONS_y=~*
+
+# PERMISSIONS=~*,connections/relational
+# PERMISSIONS=~*

packages/api/env/singledb/.env

@@ -0,0 +1,17 @@
+DEVMODE=1
+
+CONNECTIONS=mysql
+
+LABEL_mysql=MySql localhost
+SERVER_mysql=localhost
+USER_mysql=root
+PASSWORD_mysql=Pwd2020Db
+PORT_mysql=3306
+ENGINE_mysql=mysql@dbgate-plugin-mysql
+DBCONFIG_mysql=[{"name":"Chinook","connectionColor":"cyan"}]
+
+
+SINGLE_CONNECTION=mysql
+SINGLE_DATABASE=Chinook
+
+PERMISSIONS=files/charts/read

packages/api/package.json

@@ -1,13 +1,12 @@
 {
   "name": "dbgate-api",
   "main": "src/index.js",
-  "version": "3.9.5",
+  "version": "5.0.0-alpha.1",
   "homepage": "https://dbgate.org/",
   "repository": {
     "type": "git",
     "url": "https://github.com/dbgate/dbgate.git"
   },
-  "funding": "https://www.paypal.com/paypalme/JanProchazkaCz/30eur",
   "author": "Jan Prochazka",
   "license": "MIT",
   "keywords": [
@@ -18,54 +17,74 @@
     "dbgate"
   ],
   "dependencies": {
+    "activedirectory2": "^2.1.0",
     "async-lock": "^1.2.4",
-    "axios": "^0.19.0",
+    "axios": "^0.21.1",
     "body-parser": "^1.19.0",
     "bufferutil": "^4.0.1",
     "byline": "^5.0.0",
     "compare-versions": "^3.6.0",
     "cors": "^2.8.5",
     "cross-env": "^6.0.3",
-    "dbgate-sqltree": "^3.9.5",
-    "dbgate-tools": "^3.9.5",
+    "dbgate-query-splitter": "^4.9.3",
+    "dbgate-sqltree": "^5.0.0-alpha.1",
+    "dbgate-tools": "^5.0.0-alpha.1",
+    "dbgate-datalib": "^5.0.0-alpha.1",
+    "debug": "^4.3.4",
+    "diff": "^5.0.0",
+    "diff2html": "^3.4.13",
     "eslint": "^6.8.0",
     "express": "^4.17.1",
     "express-basic-auth": "^1.2.0",
     "express-fileupload": "^1.2.0",
-    "find-free-port": "^2.0.0",
-    "fs-extra": "^8.1.0",
+    "external-sorting": "^1.3.1",
+    "fs-extra": "^9.1.0",
+    "fs-reverse": "^0.0.3",
+    "get-port": "^5.1.1",
     "http": "^0.0.0",
+    "is-electron": "^2.2.1",
+    "js-yaml": "^4.1.0",
     "json-stable-stringify": "^1.0.1",
+    "jsonwebtoken": "^8.5.1",
     "line-reader": "^0.4.0",
-    "lodash": "^4.17.15",
+    "lodash": "^4.17.21",
+    "moment": "^2.24.0",
     "ncp": "^2.0.0",
-    "nedb-promises": "^4.0.1",
     "node-cron": "^2.0.3",
-    "node-ssh-forward": "^0.7.2",
+    "on-finished": "^2.4.1",
+    "pinomin": "^1.0.1",
     "portfinder": "^1.0.28",
+    "rimraf": "^3.0.0",
     "simple-encryptor": "^4.0.0",
-    "socket.io": "^2.3.0",
+    "ssh2": "^1.11.0",
     "tar": "^6.0.5",
     "uuid": "^3.4.0"
   },
   "scripts": {
-    "start": "nodemon src/index.js",
-    "start:portal": "env-cmd nodemon src/index.js",
-    "start:covid": "env-cmd -f .covid-env nodemon src/index.js",
+    "start": "env-cmd node src/index.js --listen-api",
+    "start:portal": "env-cmd -f env/portal/.env node src/index.js --listen-api",
+    "start:singledb": "env-cmd -f env/singledb/.env node src/index.js --listen-api",
+    "start:auth": "env-cmd -f env/auth/.env node src/index.js --listen-api",
+    "start:dblogin": "env-cmd -f env/dblogin/.env node src/index.js --listen-api",
+    "start:filedb": "env-cmd node src/index.js /home/jena/test/chinook/Chinook.db --listen-api",
+    "start:singleconn": "env-cmd node src/index.js --server localhost --user root --port 3307 --engine mysql@dbgate-plugin-mysql --password test --listen-api",
     "ts": "tsc",
     "build": "webpack"
   },
   "devDependencies": {
+    "@types/fs-extra": "^9.0.11",
     "@types/lodash": "^4.14.149",
-    "dbgate-types": "^3.9.5",
+    "dbgate-types": "^5.0.0-alpha.1",
     "env-cmd": "^10.1.0",
     "node-loader": "^1.0.2",
     "nodemon": "^2.0.2",
-    "typescript": "^3.7.4",
+    "typescript": "^4.4.3",
     "webpack": "^4.42.0",
     "webpack-cli": "^3.3.11"
   },
   "optionalDependencies": {
-    "msnodesqlv8": "^2.0.10"
+    "better-sqlite3": "7.6.2",
+    "msnodesqlv8": "^2.6.0",
+    "oracledb": "^5.5.0"
   }
 }

packages/api/src/controllers/apps.js

@@ -0,0 +1,280 @@
+const fs = require('fs-extra');
+const _ = require('lodash');
+const path = require('path');
+const { appdir } = require('../utility/directories');
+const socket = require('../utility/socket');
+const connections = require('./connections');
+
+module.exports = {
+  folders_meta: true,
+  async folders() {
+    const folders = await fs.readdir(appdir());
+    return [
+      ...folders.map(name => ({
+        name,
+      })),
+    ];
+  },
+
+  createFolder_meta: true,
+  async createFolder({ folder }) {
+    const name = await this.getNewAppFolder({ name: folder });
+    await fs.mkdir(path.join(appdir(), name));
+    socket.emitChanged('app-folders-changed');
+    this.emitChangedDbApp(folder);
+    return name;
+  },
+
+  files_meta: true,
+  async files({ folder }) {
+    if (!folder) return [];
+    const dir = path.join(appdir(), folder);
+    if (!(await fs.exists(dir))) return [];
+    const files = await fs.readdir(dir);
+
+    function fileType(ext, type) {
+      return files
+        .filter(name => name.endsWith(ext))
+        .map(name => ({
+          name: name.slice(0, -ext.length),
+          label: path.parse(name.slice(0, -ext.length)).base,
+          type,
+        }));
+    }
+
+    return [
+      ...fileType('.command.sql', 'command.sql'),
+      ...fileType('.query.sql', 'query.sql'),
+      ...fileType('.config.json', 'config.json'),
+    ];
+  },
+
+  async emitChangedDbApp(folder) {
+    const used = await this.getUsedAppFolders();
+    if (used.includes(folder)) {
+      socket.emitChanged('used-apps-changed');
+    }
+  },
+
+  refreshFiles_meta: true,
+  async refreshFiles({ folder }) {
+    socket.emitChanged('app-files-changed', { app: folder });
+  },
+
+  refreshFolders_meta: true,
+  async refreshFolders() {
+    socket.emitChanged(`app-folders-changed`);
+  },
+
+  deleteFile_meta: true,
+  async deleteFile({ folder, file, fileType }) {
+    await fs.unlink(path.join(appdir(), folder, `${file}.${fileType}`));
+    socket.emitChanged('app-files-changed', { app: folder });
+    this.emitChangedDbApp(folder);
+  },
+
+  renameFile_meta: true,
+  async renameFile({ folder, file, newFile, fileType }) {
+    await fs.rename(
+      path.join(path.join(appdir(), folder), `${file}.${fileType}`),
+      path.join(path.join(appdir(), folder), `${newFile}.${fileType}`)
+    );
+    socket.emitChanged('app-files-changed', { app: folder });
+    this.emitChangedDbApp(folder);
+  },
+
+  renameFolder_meta: true,
+  async renameFolder({ folder, newFolder }) {
+    const uniqueName = await this.getNewAppFolder({ name: newFolder });
+    await fs.rename(path.join(appdir(), folder), path.join(appdir(), uniqueName));
+    socket.emitChanged(`app-folders-changed`);
+  },
+
+  deleteFolder_meta: true,
+  async deleteFolder({ folder }) {
+    if (!folder) throw new Error('Missing folder parameter');
+    await fs.rmdir(path.join(appdir(), folder), { recursive: true });
+    socket.emitChanged(`app-folders-changed`);
+    socket.emitChanged('app-files-changed', { app: folder });
+    socket.emitChanged('used-apps-changed');
+  },
+
+  async getNewAppFolder({ name }) {
+    if (!(await fs.exists(path.join(appdir(), name)))) return name;
+    let index = 2;
+    while (await fs.exists(path.join(appdir(), `${name}${index}`))) {
+      index += 1;
+    }
+    return `${name}${index}`;
+  },
+
+  getUsedAppFolders_meta: true,
+  async getUsedAppFolders() {
+    const list = await connections.list();
+    const apps = [];
+
+    for (const connection of list) {
+      for (const db of connection.databases || []) {
+        for (const key of _.keys(db || {})) {
+          if (key.startsWith('useApp:') && db[key]) {
+            apps.push(key.substring('useApp:'.length));
+          }
+        }
+      }
+    }
+
+    return _.intersection(_.uniq(apps), await fs.readdir(appdir()));
+  },
+
+  getUsedApps_meta: true,
+  async getUsedApps() {
+    const apps = await this.getUsedAppFolders();
+    const res = [];
+
+    for (const folder of apps) {
+      res.push(await this.loadApp({ folder }));
+    }
+    return res;
+  },
+
+  // getAppsForDb_meta: true,
+  // async getAppsForDb({ conid, database }) {
+  //   const connection = await connections.get({ conid });
+  //   if (!connection) return [];
+  //   const db = (connection.databases || []).find(x => x.name == database);
+  //   const apps = [];
+  //   const res = [];
+  //   if (db) {
+  //     for (const key of _.keys(db || {})) {
+  //       if (key.startsWith('useApp:') && db[key]) {
+  //         apps.push(key.substring('useApp:'.length));
+  //       }
+  //     }
+  //   }
+  //   for (const folder of apps) {
+  //     res.push(await this.loadApp({ folder }));
+  //   }
+  //   return res;
+  // },
+
+  loadApp_meta: true,
+  async loadApp({ folder }) {
+    const res = {
+      queries: [],
+      commands: [],
+      name: folder,
+    };
+    const dir = path.join(appdir(), folder);
+    if (await fs.exists(dir)) {
+      const files = await fs.readdir(dir);
+
+      async function processType(ext, field) {
+        for (const file of files) {
+          if (file.endsWith(ext)) {
+            res[field].push({
+              name: file.slice(0, -ext.length),
+              sql: await fs.readFile(path.join(dir, file), { encoding: 'utf-8' }),
+            });
+          }
+        }
+      }
+
+      await processType('.command.sql', 'commands');
+      await processType('.query.sql', 'queries');
+    }
+
+    try {
+      res.virtualReferences = JSON.parse(
+        await fs.readFile(path.join(dir, 'virtual-references.config.json'), { encoding: 'utf-8' })
+      );
+    } catch (err) {
+      res.virtualReferences = [];
+    }
+    try {
+      res.dictionaryDescriptions = JSON.parse(
+        await fs.readFile(path.join(dir, 'dictionary-descriptions.config.json'), { encoding: 'utf-8' })
+      );
+    } catch (err) {
+      res.dictionaryDescriptions = [];
+    }
+
+    return res;
+  },
+
+  async saveConfigFile(appFolder, filename, filterFunc, newItem) {
+    const file = path.join(appdir(), appFolder, filename);
+
+    let json;
+    try {
+      json = JSON.parse(await fs.readFile(file, { encoding: 'utf-8' }));
+    } catch (err) {
+      json = [];
+    }
+
+    if (filterFunc) {
+      json = json.filter(filterFunc);
+    }
+
+    json = [...json, newItem];
+
+    await fs.writeFile(file, JSON.stringify(json, undefined, 2));
+
+    socket.emitChanged('app-files-changed', { app: appFolder });
+    socket.emitChanged('used-apps-changed');
+  },
+
+  saveVirtualReference_meta: true,
+  async saveVirtualReference({ appFolder, schemaName, pureName, refSchemaName, refTableName, columns }) {
+    await this.saveConfigFile(
+      appFolder,
+      'virtual-references.config.json',
+      columns.length == 1
+        ? x =>
+            !(
+              x.schemaName == schemaName &&
+              x.pureName == pureName &&
+              x.columns.length == 1 &&
+              x.columns[0].columnName == columns[0].columnName
+            )
+        : null,
+      {
+        schemaName,
+        pureName,
+        refSchemaName,
+        refTableName,
+        columns,
+      }
+    );
+    return true;
+  },
+
+  saveDictionaryDescription_meta: true,
+  async saveDictionaryDescription({ appFolder, pureName, schemaName, expression, columns, delimiter }) {
+    await this.saveConfigFile(
+      appFolder,
+      'dictionary-descriptions.config.json',
+      x => !(x.schemaName == schemaName && x.pureName == pureName),
+      {
+        schemaName,
+        pureName,
+        expression,
+        columns,
+        delimiter,
+      }
+    );
+
+    return true;
+  },
+
+  createConfigFile_meta: true,
+  async createConfigFile({ appFolder, fileName, content }) {
+    const file = path.join(appdir(), appFolder, fileName);
+    if (!(await fs.exists(file))) {
+      await fs.writeFile(file, JSON.stringify(content, undefined, 2));
+      socket.emitChanged('app-files-changed', { app: appFolder });
+      socket.emitChanged('used-apps-changed');
+      return true;
+    }
+    return false;
+  },
+};

packages/api/src/controllers/archive.js

@@ -1,15 +1,20 @@
 const fs = require('fs-extra');
-const stream = require('stream');
 const readline = require('readline');
 const path = require('path');
-const { formatWithOptions } = require('util');
-const { archivedir } = require('../utility/directories');
+const { archivedir, clearArchiveLinksCache, resolveArchiveFolder } = require('../utility/directories');
 const socket = require('../utility/socket');
-const JsonLinesDatastore = require('../utility/JsonLinesDatastore');
-const { saveFreeTableData } = require('../utility/freeTableStorage');
+const loadFilesRecursive = require('../utility/loadFilesRecursive');
+const getJslFileName = require('../utility/getJslFileName');
+const { getLogger } = require('dbgate-tools');
+const uuidv1 = require('uuid/v1');
+const dbgateApi = require('../shell');
+const jsldata = require('./jsldata');
+const platformInfo = require('../utility/platformInfo');
+
+const logger = getLogger('archive');
 
 module.exports = {
-  folders_meta: 'get',
+  folders_meta: true,
   async folders() {
     const folders = await fs.readdir(archivedir());
     return [
@@ -26,73 +31,187 @@ module.exports = {
     ];
   },
 
-  createFolder_meta: 'post',
+  createFolder_meta: true,
   async createFolder({ folder }) {
     await fs.mkdir(path.join(archivedir(), folder));
     socket.emitChanged('archive-folders-changed');
     return true;
   },
 
-  files_meta: 'get',
+  createLink_meta: true,
+  async createLink({ linkedFolder }) {
+    const folder = await this.getNewArchiveFolder({ database: path.parse(linkedFolder).name + '.link' });
+    fs.writeFile(path.join(archivedir(), folder), linkedFolder);
+    clearArchiveLinksCache();
+    socket.emitChanged('archive-folders-changed');
+    return folder;
+  },
+
+  files_meta: true,
   async files({ folder }) {
-    const dir = path.join(archivedir(), folder);
-    if (!(await fs.exists(dir))) return [];
-    const files = await fs.readdir(dir);
-    return files
-      .filter(name => name.endsWith('.jsonl'))
-      .map(name => ({
-        name: name.slice(0, -'.jsonl'.length),
-        type: 'jsonl',
-      }));
+    try {
+      const dir = resolveArchiveFolder(folder);
+      if (!(await fs.exists(dir))) return [];
+      const files = await loadFilesRecursive(dir); // fs.readdir(dir);
+
+      function fileType(ext, type) {
+        return files
+          .filter(name => name.endsWith(ext))
+          .map(name => ({
+            name: name.slice(0, -ext.length),
+            label: path.parse(name.slice(0, -ext.length)).base,
+            type,
+          }));
+      }
+
+      return [
+        ...fileType('.jsonl', 'jsonl'),
+        ...fileType('.table.yaml', 'table.yaml'),
+        ...fileType('.view.sql', 'view.sql'),
+        ...fileType('.proc.sql', 'proc.sql'),
+        ...fileType('.func.sql', 'func.sql'),
+        ...fileType('.trigger.sql', 'trigger.sql'),
+        ...fileType('.matview.sql', 'matview.sql'),
+      ];
+    } catch (err) {
+      logger.error({ err }, 'Error reading archive files');
+      return [];
+    }
   },
 
-  refreshFiles_meta: 'post',
+  refreshFiles_meta: true,
   async refreshFiles({ folder }) {
-    socket.emitChanged(`archive-files-changed-${folder}`);
-  },
-
-  refreshFolders_meta: 'post',
-  async refreshFolders() {
-    socket.emitChanged(`archive-folders-changed`);
-  },
-
-  deleteFile_meta: 'post',
-  async deleteFile({ folder, file }) {
-    await fs.unlink(path.join(archivedir(), folder, `${file}.jsonl`));
-    socket.emitChanged(`archive-files-changed-${folder}`);
-  },
-
-  deleteFolder_meta: 'post',
-  async deleteFolder({ folder }) {
-    if (!folder) throw new Error('Missing folder parameter');
-    await fs.rmdir(path.join(archivedir(), folder), { recursive: true });
-    socket.emitChanged(`archive-folders-changed`);
-  },
-
-  saveFreeTable_meta: 'post',
-  async saveFreeTable({ folder, file, data }) {
-    saveFreeTableData(path.join(archivedir(), folder, `${file}.jsonl`), data);
+    socket.emitChanged('archive-files-changed', { folder });
     return true;
   },
 
-  loadFreeTable_meta: 'post',
-  async loadFreeTable({ folder, file }) {
-    return new Promise((resolve, reject) => {
-      const fileStream = fs.createReadStream(path.join(archivedir(), folder, `${file}.jsonl`));
-      const liner = readline.createInterface({
-        input: fileStream,
-      });
-      let structure = null;
-      const rows = [];
-      liner.on('line', line => {
-        const data = JSON.parse(line);
-        if (structure) rows.push(data);
-        else structure = data;
-      });
-      liner.on('close', () => {
-        resolve({ structure, rows });
-        fileStream.close();
-      });
+  refreshFolders_meta: true,
+  async refreshFolders() {
+    socket.emitChanged(`archive-folders-changed`);
+    return true;
+  },
+
+  deleteFile_meta: true,
+  async deleteFile({ folder, file, fileType }) {
+    await fs.unlink(path.join(resolveArchiveFolder(folder), `${file}.${fileType}`));
+    socket.emitChanged(`archive-files-changed`, { folder });
+    return true;
+  },
+
+  renameFile_meta: true,
+  async renameFile({ folder, file, newFile, fileType }) {
+    await fs.rename(
+      path.join(resolveArchiveFolder(folder), `${file}.${fileType}`),
+      path.join(resolveArchiveFolder(folder), `${newFile}.${fileType}`)
+    );
+    socket.emitChanged(`archive-files-changed`, { folder });
+    return true;
+  },
+
+  modifyFile_meta: true,
+  async modifyFile({ folder, file, changeSet, mergedRows, mergeKey, mergeMode }) {
+    await jsldata.closeDataStore(`archive://${folder}/${file}`);
+    const changedFilePath = path.join(resolveArchiveFolder(folder), `${file}.jsonl`);
+
+    if (!fs.existsSync(changedFilePath)) {
+      if (!mergedRows) {
+        return false;
+      }
+      const fileStream = fs.createWriteStream(changedFilePath);
+      for (const row of mergedRows) {
+        await fileStream.write(JSON.stringify(row) + '\n');
+      }
+      await fileStream.close();
+
+      socket.emitChanged(`archive-files-changed`, { folder });
+      return true;
+    }
+
+    const tmpchangedFilePath = path.join(resolveArchiveFolder(folder), `${file}-${uuidv1()}.jsonl`);
+    const reader = await dbgateApi.modifyJsonLinesReader({
+      fileName: changedFilePath,
+      changeSet,
+      mergedRows,
+      mergeKey,
+      mergeMode,
     });
+    const writer = await dbgateApi.jsonLinesWriter({ fileName: tmpchangedFilePath });
+    await dbgateApi.copyStream(reader, writer);
+    if (platformInfo.isWindows) {
+      await fs.copyFile(tmpchangedFilePath, changedFilePath);
+      await fs.unlink(tmpchangedFilePath);
+    } else {
+      await fs.unlink(changedFilePath);
+      await fs.rename(tmpchangedFilePath, changedFilePath);
+    }
+    return true;
+  },
+
+  renameFolder_meta: true,
+  async renameFolder({ folder, newFolder }) {
+    const uniqueName = await this.getNewArchiveFolder({ database: newFolder });
+    await fs.rename(path.join(archivedir(), folder), path.join(archivedir(), uniqueName));
+    socket.emitChanged(`archive-folders-changed`);
+    return true;
+  },
+
+  deleteFolder_meta: true,
+  async deleteFolder({ folder }) {
+    if (!folder) throw new Error('Missing folder parameter');
+    if (folder.endsWith('.link')) {
+      await fs.unlink(path.join(archivedir(), folder));
+    } else {
+      await fs.rmdir(path.join(archivedir(), folder), { recursive: true });
+    }
+    socket.emitChanged(`archive-folders-changed`);
+    return true;
+  },
+
+  saveText_meta: true,
+  async saveText({ folder, file, text }) {
+    await fs.writeFile(path.join(resolveArchiveFolder(folder), `${file}.jsonl`), text);
+    socket.emitChanged(`archive-files-changed`, { folder });
+    return true;
+  },
+
+  saveJslData_meta: true,
+  async saveJslData({ folder, file, jslid, changeSet }) {
+    const source = getJslFileName(jslid);
+    const target = path.join(resolveArchiveFolder(folder), `${file}.jsonl`);
+    if (changeSet) {
+      const reader = await dbgateApi.modifyJsonLinesReader({
+        fileName: source,
+        changeSet,
+      });
+      const writer = await dbgateApi.jsonLinesWriter({ fileName: target });
+      await dbgateApi.copyStream(reader, writer);
+    } else {
+      await fs.copyFile(source, target);
+      socket.emitChanged(`archive-files-changed`, { folder });
+    }
+    return true;
+  },
+
+  saveRows_meta: true,
+  async saveRows({ folder, file, rows }) {
+    const fileStream = fs.createWriteStream(path.join(resolveArchiveFolder(folder), `${file}.jsonl`));
+    for (const row of rows) {
+      await fileStream.write(JSON.stringify(row) + '\n');
+    }
+    await fileStream.close();
+    socket.emitChanged(`archive-files-changed`, { folder });
+    return true;
+  },
+
+  async getNewArchiveFolder({ database }) {
+    const isLink = database.endsWith(database);
+    const name = isLink ? database.slice(0, -5) : database;
+    const suffix = isLink ? '.link' : '';
+    if (!(await fs.exists(path.join(archivedir(), database)))) return database;
+    let index = 2;
+    while (await fs.exists(path.join(archivedir(), `${name}${index}${suffix}`))) {
+      index += 1;
+    }
+    return `${name}${index}${suffix}`;
   },
 };

packages/api/src/controllers/auth.js

@@ -0,0 +1,150 @@
+const axios = require('axios');
+const jwt = require('jsonwebtoken');
+const getExpressPath = require('../utility/getExpressPath');
+const uuidv1 = require('uuid/v1');
+const { getLogins } = require('../utility/hasPermission');
+const { getLogger } = require('dbgate-tools');
+const AD = require('activedirectory2').promiseWrapper;
+
+const logger = getLogger('auth');
+
+const tokenSecret = uuidv1();
+
+function shouldAuthorizeApi() {
+  const logins = getLogins();
+  return !!process.env.OAUTH_AUTH || !!process.env.AD_URL || (!!logins && !process.env.BASIC_AUTH);
+}
+
+function getTokenLifetime() {
+  return process.env.TOKEN_LIFETIME || '1d';
+}
+
+function unauthorizedResponse(req, res, text) {
+  // if (req.path == getExpressPath('/config/get-settings')) {
+  //   return res.json({});
+  // }
+  // if (req.path == getExpressPath('/connections/list')) {
+  //   return res.json([]);
+  // }
+  return res.sendStatus(401).send(text);
+}
+
+function authMiddleware(req, res, next) {
+  const SKIP_AUTH_PATHS = ['/config/get', '/auth/oauth-token', '/auth/login', '/stream'];
+
+  if (!shouldAuthorizeApi()) {
+    return next();
+  }
+  let skipAuth = !!SKIP_AUTH_PATHS.find(x => req.path == getExpressPath(x));
+
+  const authHeader = req.headers.authorization;
+  if (!authHeader) {
+    if (skipAuth) {
+      return next();
+    }
+    return unauthorizedResponse(req, res, 'missing authorization header');
+  }
+  const token = authHeader.split(' ')[1];
+  try {
+    const decoded = jwt.verify(token, tokenSecret);
+    req.user = decoded;
+    return next();
+  } catch (err) {
+    if (skipAuth) {
+      return next();
+    }
+
+    logger.error({ err }, 'Sending invalid token error');
+
+    return unauthorizedResponse(req, res, 'invalid token');
+  }
+}
+
+module.exports = {
+  oauthToken_meta: true,
+  async oauthToken(params) {
+    const { redirectUri, code } = params;
+
+    const scopeParam = process.env.OAUTH_SCOPE ? `&scope=${process.env.OAUTH_SCOPE}` : '';
+    const resp = await axios.default.post(
+      `${process.env.OAUTH_TOKEN}`,
+      `grant_type=authorization_code&code=${encodeURIComponent(code)}&redirect_uri=${encodeURIComponent(
+        redirectUri
+      )}&client_id=${process.env.OAUTH_CLIENT_ID}&client_secret=${process.env.OAUTH_CLIENT_SECRET}${scopeParam}`
+    );
+
+    const { access_token, refresh_token } = resp.data;
+
+    const payload = jwt.decode(access_token);
+
+    logger.info({ payload }, 'User payload returned from OAUTH');
+
+    const login =
+      process.env.OAUTH_LOGIN_FIELD && payload && payload[process.env.OAUTH_LOGIN_FIELD]
+        ? payload[process.env.OAUTH_LOGIN_FIELD]
+        : 'oauth';
+
+    if (
+      process.env.OAUTH_ALLOWED_LOGINS &&
+      !process.env.OAUTH_ALLOWED_LOGINS.split(',').find(x => x.toLowerCase().trim() == login.toLowerCase().trim())
+    ) {
+      return { error: `Username ${login} not allowed to log in` };
+    }
+    if (access_token) {
+      return {
+        accessToken: jwt.sign({ login }, tokenSecret, { expiresIn: getTokenLifetime() }),
+      };
+    }
+
+    return { error: 'Token not found' };
+  },
+  login_meta: true,
+  async login(params) {
+    const { login, password } = params;
+
+    if (process.env.AD_URL) {
+      const adConfig = {
+        url: process.env.AD_URL,
+        baseDN: process.env.AD_BASEDN,
+        username: process.env.AD_USERNAME,
+        password: process.env.AD_PASSOWRD,
+      };
+      const ad = new AD(adConfig);
+      try {
+        const res = await ad.authenticate(login, password);
+        if (!res) {
+          return { error: 'Login failed' };
+        }
+        if (
+          process.env.AD_ALLOWED_LOGINS &&
+          !process.env.AD_ALLOWED_LOGINS.split(',').find(x => x.toLowerCase().trim() == login.toLowerCase().trim())
+        ) {
+          return { error: `Username ${login} not allowed to log in` };
+        }
+        return {
+          accessToken: jwt.sign({ login }, tokenSecret, { expiresIn: getTokenLifetime() }),
+        };
+      } catch (err) {
+        logger.error({ err }, 'Failed active directory authentization');
+        return {
+          error: err.message,
+        };
+      }
+    }
+
+    const logins = getLogins();
+    if (!logins) {
+      return { error: 'Logins not configured' };
+    }
+    const foundLogin = logins.find(x => x.login == login);
+    if (foundLogin && foundLogin.password == password) {
+      return {
+        accessToken: jwt.sign({ login }, tokenSecret, { expiresIn: getTokenLifetime() }),
+      };
+    }
+    return { error: 'Invalid credentials' };
+  },
+
+  authMiddleware,
+  shouldAuthorizeApi,
+};

packages/api/src/controllers/config.js

@@ -1,41 +1,133 @@
+const fs = require('fs-extra');
+const os = require('os');
+const path = require('path');
+const axios = require('axios');
+const { datadir, getLogsFilePath } = require('../utility/directories');
+const { hasPermission, getLogins } = require('../utility/hasPermission');
+const socket = require('../utility/socket');
+const _ = require('lodash');
+const AsyncLock = require('async-lock');
+
 const currentVersion = require('../currentVersion');
 const platformInfo = require('../utility/platformInfo');
+const connections = require('../controllers/connections');
+
+const lock = new AsyncLock();
 
 module.exports = {
-  get_meta: 'get',
-  async get() {
-    // const toolbarButtons = process.env.TOOLBAR;
-    // const toolbar = toolbarButtons
-    //   ? toolbarButtons.split(',').map((name) => ({
-    //       name,
-    //       icon: process.env[`ICON_${name}`],
-    //       title: process.env[`TITLE_${name}`],
-    //       page: process.env[`PAGE_${name}`],
-    //     }))
-    //   : null;
-    // const startupPages = process.env.STARTUP_PAGES ? process.env.STARTUP_PAGES.split(',') : [];
-    const permissions = process.env.PERMISSIONS ? process.env.PERMISSIONS.split(',') : null;
-    const singleDatabase =
-      process.env.SINGLE_CONNECTION && process.env.SINGLE_DATABASE
-        ? {
-            conid: process.env.SINGLE_CONNECTION,
-            database: process.env.SINGLE_DATABASE,
-          }
-        : null;
+  // settingsValue: {},
+
+  // async _init() {
+  //   try {
+  //     this.settingsValue = JSON.parse(await fs.readFile(path.join(datadir(), 'settings.json'), { encoding: 'utf-8' }));
+  //   } catch (err) {
+  //     this.settingsValue = {};
+  //   }
+  // },
+
+  get_meta: true,
+  async get(_params, req) {
+    const logins = getLogins();
+    const loginName =
+      req && req.user && req.user.login ? req.user.login : req && req.auth && req.auth.user ? req.auth.user : null;
+    const login = logins && loginName ? logins.find(x => x.login == loginName) : null;
+    const permissions = login ? login.permissions : process.env.PERMISSIONS;
 
     return {
-      runAsPortal: !!process.env.CONNECTIONS,
-      // toolbar,
-      // startupPages,
-      singleDatabase,
+      runAsPortal: !!connections.portalConnections,
+      singleDbConnection: connections.singleDbConnection,
+      singleConnection: connections.singleConnection,
+      // hideAppEditor: !!process.env.HIDE_APP_EDITOR,
+      allowShellConnection: platformInfo.allowShellConnection,
+      allowShellScripting: platformInfo.allowShellScripting,
+      isDocker: platformInfo.isDocker,
       permissions,
+      login,
+      oauth: process.env.OAUTH_AUTH,
+      oauthClient: process.env.OAUTH_CLIENT_ID,
+      oauthScope: process.env.OAUTH_SCOPE,
+      oauthLogout: process.env.OAUTH_LOGOUT,
+      isLoginForm: !!process.env.AD_URL || (!!logins && !process.env.BASIC_AUTH),
+      logsFilePath: getLogsFilePath(),
+      connectionsFilePath: path.join(datadir(), 'connections.jsonl'),
       ...currentVersion,
     };
   },
 
-  platformInfo_meta: 'get',
+  logout_meta: {
+    method: 'get',
+    raw: true,
+  },
+  logout(req, res) {
+    res.status(401).send('Logged out<br><a href="../..">Back to DbGate</a>');
+  },
+
+  platformInfo_meta: true,
   async platformInfo() {
     return platformInfo;
   },
-  
+
+  getSettings_meta: true,
+  async getSettings() {
+    const res = await lock.acquire('settings', async () => {
+      return await this.loadSettings();
+    });
+    return res;
+  },
+
+  fillMissingSettings(value) {
+    const res = {
+      ...value,
+    };
+    if (value['app.useNativeMenu'] !== true && value['app.useNativeMenu'] !== false) {
+      // res['app.useNativeMenu'] = os.platform() == 'darwin' ? true : false;
+      res['app.useNativeMenu'] = false;
+    }
+    for (const envVar in process.env) {
+      if (envVar.startsWith('SETTINGS_')) {
+        const key = envVar.substring('SETTINGS_'.length);
+        if (!res[key]) {
+          res[key] = process.env[envVar];
+        }
+      }
+    }
+    return res;
+  },
+
+  async loadSettings() {
+    try {
+      const settingsText = await fs.readFile(path.join(datadir(), 'settings.json'), { encoding: 'utf-8' });
+      return this.fillMissingSettings(JSON.parse(settingsText));
+    } catch (err) {
+      return this.fillMissingSettings({});
+    }
+  },
+
+  updateSettings_meta: true,
+  async updateSettings(values, req) {
+    if (!hasPermission(`settings/change`, req)) return false;
+
+    const res = await lock.acquire('settings', async () => {
+      const currentValue = await this.loadSettings();
+      try {
+        const updated = {
+          ...currentValue,
+          ...values,
+        };
+        await fs.writeFile(path.join(datadir(), 'settings.json'), JSON.stringify(updated, undefined, 2));
+        // this.settingsValue = updated;
+        socket.emitChanged(`settings-changed`);
+        return updated;
+      } catch (err) {
+        return false;
+      }
+    });
+    return res;
+  },
+
+  changelog_meta: true,
+  async changelog() {
+    const resp = await axios.default.get('https://raw.githubusercontent.com/dbgate/dbgate/master/CHANGELOG.md');
+    return resp.data;
+  },
 };

packages/api/src/controllers/connections.js

@@ -1,89 +1,366 @@
 const path = require('path');
 const { fork } = require('child_process');
 const _ = require('lodash');
-const nedb = require('nedb-promises');
+const fs = require('fs-extra');
+const crypto = require('crypto');
 
-const { datadir } = require('../utility/directories');
+const { datadir, filesdir } = require('../utility/directories');
 const socket = require('../utility/socket');
-const { encryptConnection } = require('../utility/crypting');
+const { encryptConnection, maskConnection } = require('../utility/crypting');
 const { handleProcessCommunication } = require('../utility/processComm');
+const { pickSafeConnectionInfo } = require('../utility/crypting');
+const JsonLinesDatabase = require('../utility/JsonLinesDatabase');
+
+const processArgs = require('../utility/processArgs');
+const { safeJsonParse, getLogger } = require('dbgate-tools');
+const platformInfo = require('../utility/platformInfo');
+const { connectionHasPermission, testConnectionPermission } = require('../utility/hasPermission');
+const pipeForkLogs = require('../utility/pipeForkLogs');
+
+const logger = getLogger('connections');
+
+let volatileConnections = {};
+
+function getNamedArgs() {
+  const res = {};
+  for (let i = 0; i < process.argv.length; i++) {
+    const name = process.argv[i];
+    if (name.startsWith('--')) {
+      let value = process.argv[i + 1];
+      if (value && value.startsWith('--')) value = null;
+      res[name.substring(2)] = value == null ? true : value;
+      i++;
+    } else {
+      if (name.endsWith('.db') || name.endsWith('.sqlite') || name.endsWith('.sqlite3')) {
+        res.databaseFile = name;
+        res.engine = 'sqlite@dbgate-plugin-sqlite';
+      }
+    }
+  }
+  return res;
+}
+
+function getDatabaseFileLabel(databaseFile) {
+  if (!databaseFile) return databaseFile;
+  const m = databaseFile.match(/[\/]([^\/]+)$/);
+  if (m) return m[1];
+  return databaseFile;
+}
 
 function getPortalCollections() {
   if (process.env.CONNECTIONS) {
-    return _.compact(process.env.CONNECTIONS.split(',')).map(id => ({
+    const connections = _.compact(process.env.CONNECTIONS.split(',')).map(id => ({
       _id: id,
       engine: process.env[`ENGINE_${id}`],
       server: process.env[`SERVER_${id}`],
       user: process.env[`USER_${id}`],
       password: process.env[`PASSWORD_${id}`],
+      passwordMode: process.env[`PASSWORD_MODE_${id}`],
       port: process.env[`PORT_${id}`],
+      databaseUrl: process.env[`URL_${id}`],
+      useDatabaseUrl: !!process.env[`URL_${id}`],
+      databaseFile: process.env[`FILE_${id}`],
+      socketPath: process.env[`SOCKET_PATH_${id}`],
+      authType: process.env[`AUTH_TYPE_${id}`] || (process.env[`SOCKET_PATH_${id}`] ? 'socket' : undefined),
+      defaultDatabase:
+        process.env[`DATABASE_${id}`] ||
+        (process.env[`FILE_${id}`] ? getDatabaseFileLabel(process.env[`FILE_${id}`]) : null),
+      singleDatabase: !!process.env[`DATABASE_${id}`] || !!process.env[`FILE_${id}`],
       displayName: process.env[`LABEL_${id}`],
+      isReadOnly: process.env[`READONLY_${id}`],
+      databases: process.env[`DBCONFIG_${id}`] ? safeJsonParse(process.env[`DBCONFIG_${id}`]) : null,
+      parent: process.env[`PARENT_${id}`] || undefined,
+
+      // SSH tunnel
+      useSshTunnel: process.env[`USE_SSH_${id}`],
+      sshHost: process.env[`SSH_HOST_${id}`],
+      sshPort: process.env[`SSH_PORT_${id}`],
+      sshMode: process.env[`SSH_MODE_${id}`],
+      sshLogin: process.env[`SSH_LOGIN_${id}`],
+      sshPassword: process.env[`SSH_PASSWORD_${id}`],
+      sshKeyfile: process.env[`SSH_KEY_FILE_${id}`],
+      sshKeyfilePassword: process.env[`SSH_KEY_FILE_PASSWORD_${id}`],
+
+      // SSL
+      useSsl: process.env[`USE_SSL_${id}`],
+      sslCaFile: process.env[`SSL_CA_FILE_${id}`],
+      sslCertFile: process.env[`SSL_CERT_FILE_${id}`],
+      sslCertFilePassword: process.env[`SSL_CERT_FILE_PASSWORD_${id}`],
+      sslKeyFile: process.env[`SSL_KEY_FILE_${id}`],
+      sslRejectUnauthorized: process.env[`SSL_REJECT_UNAUTHORIZED_${id}`],
     }));
+
+    logger.info({ connections: connections.map(pickSafeConnectionInfo) }, 'Using connections from ENV variables');
+    const noengine = connections.filter(x => !x.engine);
+    if (noengine.length > 0) {
+      logger.warn(
+        { connections: noengine.map(x => x._id) },
+        'Invalid CONNECTIONS configutation, missing ENGINE for connection ID'
+      );
+    }
+    return connections;
+  }
+
+  const args = getNamedArgs();
+  if (args.databaseFile) {
+    return [
+      {
+        _id: 'argv',
+        databaseFile: args.databaseFile,
+        singleDatabase: true,
+        defaultDatabase: getDatabaseFileLabel(args.databaseFile),
+        engine: args.engine,
+      },
+    ];
+  }
+  if (args.databaseUrl) {
+    return [
+      {
+        _id: 'argv',
+        useDatabaseUrl: true,
+        ...args,
+      },
+    ];
+  }
+  if (args.server) {
+    return [
+      {
+        _id: 'argv',
+        ...args,
+      },
+    ];
+  }
+
+  return null;
+}
+
+const portalConnections = getPortalCollections();
+
+function getSingleDbConnection() {
+  if (process.env.SINGLE_CONNECTION && process.env.SINGLE_DATABASE) {
+    // @ts-ignore
+    const connection = portalConnections.find(x => x._id == process.env.SINGLE_CONNECTION);
+    return {
+      connection,
+      name: process.env.SINGLE_DATABASE,
+    };
+  }
+  // @ts-ignore
+  const arg0 = (portalConnections || []).find(x => x._id == 'argv');
+  if (arg0) {
+    // @ts-ignore
+    if (arg0.singleDatabase) {
+      return {
+        connection: arg0,
+        // @ts-ignore
+        name: arg0.defaultDatabase,
+      };
+    }
   }
   return null;
 }
-const portalConnections = getPortalCollections();
+
+function getSingleConnection() {
+  if (getSingleDbConnection()) return null;
+  if (process.env.SINGLE_CONNECTION) {
+    // @ts-ignore
+    const connection = portalConnections.find(x => x._id == process.env.SINGLE_CONNECTION);
+    if (connection) {
+      return connection;
+    }
+  }
+  // @ts-ignore
+  const arg0 = (portalConnections || []).find(x => x._id == 'argv');
+  if (arg0) {
+    return arg0;
+  }
+  return null;
+}
+
+const singleDbConnection = getSingleDbConnection();
+const singleConnection = getSingleConnection();
 
 module.exports = {
   datastore: null,
   opened: [],
+  singleDbConnection,
+  singleConnection,
+  portalConnections,
 
   async _init() {
     const dir = datadir();
     if (!portalConnections) {
       // @ts-ignore
-      this.datastore = nedb.create(path.join(dir, 'connections.jsonl'));
+      this.datastore = new JsonLinesDatabase(path.join(dir, 'connections.jsonl'));
     }
   },
 
-  list_meta: 'get',
-  async list() {
-    return portalConnections || this.datastore.find();
+  list_meta: true,
+  async list(_params, req) {
+    if (portalConnections) {
+      if (platformInfo.allowShellConnection) return portalConnections;
+      return portalConnections.map(maskConnection).filter(x => connectionHasPermission(x, req));
+    }
+    return (await this.datastore.find()).filter(x => connectionHasPermission(x, req));
   },
 
-  test_meta: {
-    method: 'post',
-    raw: true,
-  },
-  test(req, res) {
-    const subprocess = fork(process.argv[1], ['connectProcess', ...process.argv.slice(3)]);
-    subprocess.on('message', resp => {
-      if (handleProcessCommunication(resp, subprocess)) return;
-      // @ts-ignore
-      const { msgtype } = resp;
-      if (msgtype == 'connected' || msgtype == 'error') {
-        res.json(resp);
+  test_meta: true,
+  test(connection) {
+    const subprocess = fork(
+      global['API_PACKAGE'] || process.argv[1],
+      [
+        '--is-forked-api',
+        '--start-process',
+        'connectProcess',
+        ...processArgs.getPassArgs(),
+        // ...process.argv.slice(3),
+      ],
+      {
+        stdio: ['ignore', 'pipe', 'pipe', 'ipc'],
       }
+    );
+    pipeForkLogs(subprocess);
+    subprocess.send(connection);
+    return new Promise(resolve => {
+      subprocess.on('message', resp => {
+        if (handleProcessCommunication(resp, subprocess)) return;
+        // @ts-ignore
+        const { msgtype } = resp;
+        if (msgtype == 'connected' || msgtype == 'error') {
+          resolve(resp);
+        }
+      });
     });
-    subprocess.send(req.body);
   },
 
-  save_meta: 'post',
+  saveVolatile_meta: true,
+  async saveVolatile({ conid, user, password, test }) {
+    const old = await this.getCore({ conid });
+    const res = {
+      ...old,
+      _id: crypto.randomUUID(),
+      password,
+      passwordMode: undefined,
+      unsaved: true,
+    };
+    if (old.passwordMode == 'askUser') {
+      res.user = user;
+    }
+
+    if (test) {
+      const testRes = await this.test(res);
+      if (testRes.msgtype == 'connected') {
+        volatileConnections[res._id] = res;
+        return {
+          ...res,
+          msgtype: 'connected',
+        };
+      }
+      return testRes;
+    } else {
+      volatileConnections[res._id] = res;
+      return res;
+    }
+  },
+
+  save_meta: true,
   async save(connection) {
     if (portalConnections) return;
     let res;
     const encrypted = encryptConnection(connection);
     if (connection._id) {
-      res = await this.datastore.update(_.pick(connection, '_id'), encrypted);
+      res = await this.datastore.update(encrypted);
     } else {
       res = await this.datastore.insert(encrypted);
     }
     socket.emitChanged('connection-list-changed');
+    socket.emitChanged('used-apps-changed');
+    if (this._closeAll) {
+      this._closeAll(connection._id);
+    }
+    // for (const db of connection.databases || []) {
+    //   socket.emitChanged(`db-apps-changed-${connection._id}-${db.name}`);
+    // }
     return res;
   },
 
-  delete_meta: 'post',
-  async delete(connection) {
+  update_meta: true,
+  async update({ _id, values }, req) {
     if (portalConnections) return;
-    const res = await this.datastore.remove(_.pick(connection, '_id'));
+    testConnectionPermission(_id, req);
+    const res = await this.datastore.patch(_id, values);
     socket.emitChanged('connection-list-changed');
     return res;
   },
 
-  get_meta: 'get',
-  async get({ conid }) {
-    if (portalConnections) return portalConnections.find(x => x._id == conid);
-    const res = await this.datastore.find({ _id: conid });
-    return res[0];
+  batchChangeFolder_meta: true,
+  async batchChangeFolder({ folder, newFolder }, req) {
+    // const updated = await this.datastore.find(x => x.parent == folder);
+    const res = await this.datastore.updateAll(x => (x.parent == folder ? { ...x, parent: newFolder } : x));
+    socket.emitChanged('connection-list-changed');
+    return res;
+  },
+
+  updateDatabase_meta: true,
+  async updateDatabase({ conid, database, values }, req) {
+    if (portalConnections) return;
+    testConnectionPermission(conid, req);
+    const conn = await this.datastore.get(conid);
+    let databases = (conn && conn.databases) || [];
+    if (databases.find(x => x.name == database)) {
+      databases = databases.map(x => (x.name == database ? { ...x, ...values } : x));
+    } else {
+      databases = [...databases, { name: database, ...values }];
+    }
+    const res = await this.datastore.patch(conid, { databases });
+    socket.emitChanged('connection-list-changed');
+    socket.emitChanged('used-apps-changed');
+    // socket.emitChanged(`db-apps-changed-${conid}-${database}`);
+    return res;
+  },
+
+  delete_meta: true,
+  async delete(connection, req) {
+    if (portalConnections) return;
+    testConnectionPermission(connection, req);
+    const res = await this.datastore.remove(connection._id);
+    socket.emitChanged('connection-list-changed');
+    return res;
+  },
+
+  async getCore({ conid, mask = false }) {
+    if (!conid) return null;
+    const volatile = volatileConnections[conid];
+    if (volatile) {
+      return volatile;
+    }
+    if (portalConnections) {
+      const res = portalConnections.find(x => x._id == conid) || null;
+      return mask && !platformInfo.allowShellConnection ? maskConnection(res) : res;
+    }
+    const res = await this.datastore.get(conid);
+    return res || null;
+  },
+
+  get_meta: true,
+  async get({ conid }, req) {
+    testConnectionPermission(conid, req);
+    return this.getCore({ conid, mask: true });
+  },
+
+  newSqliteDatabase_meta: true,
+  async newSqliteDatabase({ file }) {
+    const sqliteDir = path.join(filesdir(), 'sqlite');
+    if (!(await fs.exists(sqliteDir))) {
+      await fs.mkdir(sqliteDir);
+    }
+    const databaseFile = path.join(sqliteDir, `${file}.sqlite`);
+    const res = await this.save({
+      engine: 'sqlite@dbgate-plugin-sqlite',
+      databaseFile,
+      singleDatabase: true,
+      defaultDatabase: `${file}.sqlite`,
+    });
+    return res;
   },
 };

packages/api/src/controllers/databaseConnections.js

@@ -1,9 +1,37 @@
 const uuidv1 = require('uuid/v1');
 const connections = require('./connections');
+const archive = require('./archive');
 const socket = require('../utility/socket');
 const { fork } = require('child_process');
-const { DatabaseAnalyser } = require('dbgate-tools');
+const {
+  DatabaseAnalyser,
+  computeDbDiffRows,
+  getCreateObjectScript,
+  getAlterDatabaseScript,
+  generateDbPairingId,
+  matchPairedObjects,
+  extendDatabaseInfo,
+  modelCompareDbDiffOptions,
+  getLogger,
+} = require('dbgate-tools');
+const { html, parse } = require('diff2html');
 const { handleProcessCommunication } = require('../utility/processComm');
+const config = require('./config');
+const fs = require('fs-extra');
+const exportDbModel = require('../utility/exportDbModel');
+const { archivedir, resolveArchiveFolder, uploadsdir } = require('../utility/directories');
+const path = require('path');
+const importDbModel = require('../utility/importDbModel');
+const requireEngineDriver = require('../utility/requireEngineDriver');
+const generateDeploySql = require('../shell/generateDeploySql');
+const { createTwoFilesPatch } = require('diff');
+const diff2htmlPage = require('../utility/diff2htmlPage');
+const processArgs = require('../utility/processArgs');
+const { testConnectionPermission } = require('../utility/hasPermission');
+const { MissingCredentialsError } = require('../utility/exceptions');
+const pipeForkLogs = require('../utility/pipeForkLogs');
+
+const logger = getLogger('databaseConnections');
 
 module.exports = {
   /** @type {import('dbgate-types').OpenedDatabaseConnection[]} */
@@ -11,15 +39,32 @@ module.exports = {
   closed: {},
   requests: {},
 
+  async _init() {
+    connections._closeAll = conid => this.closeAll(conid);
+  },
+
   handle_structure(conid, database, { structure }) {
     const existing = this.opened.find(x => x.conid == conid && x.database == database);
     if (!existing) return;
     existing.structure = structure;
-    socket.emitChanged(`database-structure-changed-${conid}-${database}`);
+    socket.emitChanged('database-structure-changed', { conid, database });
   },
+  handle_structureTime(conid, database, { analysedTime }) {
+    const existing = this.opened.find(x => x.conid == conid && x.database == database);
+    if (!existing) return;
+    existing.analysedTime = analysedTime;
+    socket.emitChanged(`database-status-changed`, { conid, database });
+  },
+  handle_version(conid, database, { version }) {
+    const existing = this.opened.find(x => x.conid == conid && x.database == database);
+    if (!existing) return;
+    existing.serverVersion = version;
+    socket.emitChanged(`database-server-version-changed`, { conid, database });
+  },
+
   handle_error(conid, database, props) {
     const { error } = props;
-    console.log(`Error in database connection ${conid}, database ${database}: ${error}`);
+    logger.error(`Error in database connection ${conid}, database ${database}: ${error}`);
   },
   handle_response(conid, database, { msgid, ...response }) {
     const [resolve, reject] = this.requests[msgid];
@@ -27,11 +72,12 @@ module.exports = {
     delete this.requests[msgid];
   },
   handle_status(conid, database, { status }) {
+    // console.log('HANDLE SET STATUS', status);
     const existing = this.opened.find(x => x.conid == conid && x.database == database);
     if (!existing) return;
-    if (existing.status == status) return;
+    if (existing.status && status && existing.status.counter > status.counter) return;
     existing.status = status;
-    socket.emitChanged(`database-status-changed-${conid}-${database}`);
+    socket.emitChanged(`database-status-changed`, { conid, database });
   },
 
   handle_ping() {},
@@ -39,14 +85,31 @@ module.exports = {
   async ensureOpened(conid, database) {
     const existing = this.opened.find(x => x.conid == conid && x.database == database);
     if (existing) return existing;
-    const connection = await connections.get({ conid });
-    const subprocess = fork(process.argv[1], ['databaseConnectionProcess', ...process.argv.slice(3)]);
+    const connection = await connections.getCore({ conid });
+    if (connection.passwordMode == 'askPassword' || connection.passwordMode == 'askUser') {
+      throw new MissingCredentialsError({ conid, passwordMode: connection.passwordMode });
+    }
+    const subprocess = fork(
+      global['API_PACKAGE'] || process.argv[1],
+      [
+        '--is-forked-api',
+        '--start-process',
+        'databaseConnectionProcess',
+        ...processArgs.getPassArgs(),
+        // ...process.argv.slice(3),
+      ],
+      {
+        stdio: ['ignore', 'pipe', 'pipe', 'ipc'],
+      }
+    );
+    pipeForkLogs(subprocess);
     const lastClosed = this.closed[`${conid}/${database}`];
     const newOpened = {
       conid,
       database,
       subprocess,
       structure: lastClosed ? lastClosed.structure : DatabaseAnalyser.createEmptyStructure(),
+      serverVersion: lastClosed ? lastClosed.serverVersion : null,
       connection,
       status: { name: 'pending' },
     };
@@ -67,6 +130,7 @@ module.exports = {
       msgtype: 'connect',
       connection: { ...connection, database },
       structure: lastClosed ? lastClosed.structure : null,
+      globalSettings: await config.getSettings(),
     });
     return newOpened;
   },
@@ -76,14 +140,20 @@ module.exports = {
     const msgid = uuidv1();
     const promise = new Promise((resolve, reject) => {
       this.requests[msgid] = [resolve, reject];
-      conn.subprocess.send({ msgid, ...message });
+      try {
+        conn.subprocess.send({ msgid, ...message });
+      } catch (err) {
+        logger.error({ err }, 'Error sending request do process');
+        this.close(conn.conid, conn.database);
+      }
     });
     return promise;
   },
 
-  queryData_meta: 'post',
-  async queryData({ conid, database, sql }) {
-    console.log(`Processing query, conid=${conid}, database=${database}, sql=${sql}`);
+  queryData_meta: true,
+  async queryData({ conid, database, sql }, req) {
+    testConnectionPermission(conid, req);
+    logger.info({ conid, database, sql }, 'Processing query');
     const opened = await this.ensureOpened(conid, database);
     // if (opened && opened.status && opened.status.name == 'error') {
     //   return opened.status;
@@ -92,43 +162,186 @@ module.exports = {
     return res;
   },
 
-  status_meta: 'get',
-  async status({ conid, database }) {
+  sqlSelect_meta: true,
+  async sqlSelect({ conid, database, select }, req) {
+    testConnectionPermission(conid, req);
+    const opened = await this.ensureOpened(conid, database);
+    const res = await this.sendRequest(opened, { msgtype: 'sqlSelect', select });
+    return res;
+  },
+
+  runScript_meta: true,
+  async runScript({ conid, database, sql, useTransaction }, req) {
+    testConnectionPermission(conid, req);
+    logger.info({ conid, database, sql }, 'Processing script');
+    const opened = await this.ensureOpened(conid, database);
+    const res = await this.sendRequest(opened, { msgtype: 'runScript', sql, useTransaction });
+    return res;
+  },
+
+  collectionData_meta: true,
+  async collectionData({ conid, database, options }, req) {
+    testConnectionPermission(conid, req);
+    const opened = await this.ensureOpened(conid, database);
+    const res = await this.sendRequest(opened, { msgtype: 'collectionData', options });
+    return res.result || null;
+  },
+
+  async loadDataCore(msgtype, { conid, database, ...args }, req) {
+    testConnectionPermission(conid, req);
+    const opened = await this.ensureOpened(conid, database);
+    const res = await this.sendRequest(opened, { msgtype, ...args });
+    if (res.errorMessage) {
+      console.error(res.errorMessage);
+
+      return {
+        errorMessage: res.errorMessage,
+      };
+    }
+    return res.result || null;
+  },
+
+  loadKeys_meta: true,
+  async loadKeys({ conid, database, root, filter }, req) {
+    testConnectionPermission(conid, req);
+    return this.loadDataCore('loadKeys', { conid, database, root, filter });
+  },
+
+  exportKeys_meta: true,
+  async exportKeys({ conid, database, options }, req) {
+    testConnectionPermission(conid, req);
+    return this.loadDataCore('exportKeys', { conid, database, options });
+  },
+
+  loadKeyInfo_meta: true,
+  async loadKeyInfo({ conid, database, key }, req) {
+    testConnectionPermission(conid, req);
+    return this.loadDataCore('loadKeyInfo', { conid, database, key });
+  },
+
+  loadKeyTableRange_meta: true,
+  async loadKeyTableRange({ conid, database, key, cursor, count }, req) {
+    testConnectionPermission(conid, req);
+    return this.loadDataCore('loadKeyTableRange', { conid, database, key, cursor, count });
+  },
+
+  loadFieldValues_meta: true,
+  async loadFieldValues({ conid, database, schemaName, pureName, field, search }, req) {
+    testConnectionPermission(conid, req);
+    return this.loadDataCore('loadFieldValues', { conid, database, schemaName, pureName, field, search });
+  },
+
+  callMethod_meta: true,
+  async callMethod({ conid, database, method, args }, req) {
+    testConnectionPermission(conid, req);
+    return this.loadDataCore('callMethod', { conid, database, method, args });
+
+    // const opened = await this.ensureOpened(conid, database);
+    // const res = await this.sendRequest(opened, { msgtype: 'callMethod', method, args });
+    // if (res.errorMessage) {
+    //   console.error(res.errorMessage);
+    // }
+    // return res.result || null;
+  },
+
+  updateCollection_meta: true,
+  async updateCollection({ conid, database, changeSet }, req) {
+    testConnectionPermission(conid, req);
+    const opened = await this.ensureOpened(conid, database);
+    const res = await this.sendRequest(opened, { msgtype: 'updateCollection', changeSet });
+    if (res.errorMessage) {
+      return {
+        errorMessage: res.errorMessage,
+      };
+    }
+    return res.result || null;
+  },
+
+  status_meta: true,
+  async status({ conid, database }, req) {
+    if (!conid) {
+      return {
+        name: 'error',
+        message: 'No connection',
+      };
+    }
+    testConnectionPermission(conid, req);
     const existing = this.opened.find(x => x.conid == conid && x.database == database);
-    if (existing) return existing.status;
+    if (existing) {
+      return {
+        ...existing.status,
+        analysedTime: existing.analysedTime,
+      };
+    }
     const lastClosed = this.closed[`${conid}/${database}`];
-    if (lastClosed) return lastClosed.status;
+    if (lastClosed) {
+      return {
+        ...lastClosed.status,
+        analysedTime: lastClosed.analysedTime,
+      };
+    }
     return {
       name: 'error',
       message: 'Not connected',
     };
   },
 
-  ping_meta: 'post',
-  async ping({ conid, database }) {
-    const existing = this.opened.find(x => x.conid == conid && x.database == database);
+  ping_meta: true,
+  async ping({ conid, database }, req) {
+    testConnectionPermission(conid, req);
+    let existing = this.opened.find(x => x.conid == conid && x.database == database);
+
     if (existing) {
-      existing.subprocess.send({ msgtype: 'ping' });
+      try {
+        existing.subprocess.send({ msgtype: 'ping' });
+      } catch (err) {
+        logger.error({ err }, 'Error pinging DB connection');
+        this.close(conid, database);
+
+        return {
+          status: 'error',
+          message: 'Ping failed',
+        };
+      }
+    } else {
+      // @ts-ignore
+      existing = await this.ensureOpened(conid, database);
     }
+
     return {
       status: 'ok',
       connectionStatus: existing ? existing.status : null,
     };
   },
 
-  refresh_meta: 'post',
-  async refresh({ conid, database }) {
-    this.close(conid, database);
+  refresh_meta: true,
+  async refresh({ conid, database, keepOpen }, req) {
+    testConnectionPermission(conid, req);
+    if (!keepOpen) this.close(conid, database);
 
     await this.ensureOpened(conid, database);
     return { status: 'ok' };
   },
 
+  syncModel_meta: true,
+  async syncModel({ conid, database, isFullRefresh }, req) {
+    testConnectionPermission(conid, req);
+    const conn = await this.ensureOpened(conid, database);
+    conn.subprocess.send({ msgtype: 'syncModel', isFullRefresh });
+    return { status: 'ok' };
+  },
+
   close(conid, database, kill = true) {
     const existing = this.opened.find(x => x.conid == conid && x.database == database);
     if (existing) {
       existing.disconnected = true;
-      if (kill) existing.subprocess.kill();
+      if (kill) {
+        try {
+          existing.subprocess.kill();
+        } catch (err) {
+          logger.error({ err }, 'Error killing subprocess');
+        }
+      }
       this.opened = this.opened.filter(x => x.conid != conid || x.database != database);
       this.closed[`${conid}/${database}`] = {
         status: {
@@ -137,12 +350,31 @@ module.exports = {
         },
         structure: existing.structure,
       };
-      socket.emitChanged(`database-status-changed-${conid}-${database}`);
+      socket.emitChanged(`database-status-changed`, { conid, database });
     }
   },
 
-  structure_meta: 'get',
-  async structure({ conid, database }) {
+  closeAll(conid, kill = true) {
+    for (const existing of this.opened.filter(x => x.conid == conid)) {
+      this.close(conid, existing.database, kill);
+    }
+  },
+
+  disconnect_meta: true,
+  async disconnect({ conid, database }, req) {
+    testConnectionPermission(conid, req);
+    await this.close(conid, database, true);
+    return { status: 'ok' };
+  },
+
+  structure_meta: true,
+  async structure({ conid, database }, req) {
+    testConnectionPermission(conid, req);
+    if (conid == '__model') {
+      const model = await importDbModel(database);
+      return model;
+    }
+
     const opened = await this.ensureOpened(conid, database);
     return opened.structure;
     // const existing = this.opened.find((x) => x.conid == conid && x.database == database);
@@ -153,11 +385,123 @@ module.exports = {
     // };
   },
 
-  // runCommand_meta: 'post',
+  serverVersion_meta: true,
+  async serverVersion({ conid, database }, req) {
+    if (!conid) {
+      return null;
+    }
+    testConnectionPermission(conid, req);
+    if (!conid) return null;
+    const opened = await this.ensureOpened(conid, database);
+    return opened.serverVersion || null;
+  },
+
+  sqlPreview_meta: true,
+  async sqlPreview({ conid, database, objects, options }, req) {
+    testConnectionPermission(conid, req);
+    // wait for structure
+    await this.structure({ conid, database });
+
+    const opened = await this.ensureOpened(conid, database);
+    const res = await this.sendRequest(opened, { msgtype: 'sqlPreview', objects, options });
+    return res;
+  },
+
+  exportModel_meta: true,
+  async exportModel({ conid, database }, req) {
+    testConnectionPermission(conid, req);
+    const archiveFolder = await archive.getNewArchiveFolder({ database });
+    await fs.mkdir(path.join(archivedir(), archiveFolder));
+    const model = await this.structure({ conid, database });
+    await exportDbModel(model, path.join(archivedir(), archiveFolder));
+    socket.emitChanged(`archive-folders-changed`);
+    return { archiveFolder };
+  },
+
+  generateDeploySql_meta: true,
+  async generateDeploySql({ conid, database, archiveFolder }, req) {
+    testConnectionPermission(conid, req);
+    const opened = await this.ensureOpened(conid, database);
+    const res = await this.sendRequest(opened, {
+      msgtype: 'generateDeploySql',
+      modelFolder: resolveArchiveFolder(archiveFolder),
+    });
+    return res;
+
+    // const connection = await connections.get({ conid });
+    // return generateDeploySql({
+    //   connection,
+    //   analysedStructure: await this.structure({ conid, database }),
+    //   modelFolder: resolveArchiveFolder(archiveFolder),
+    // });
+
+    // const deployedModel = generateDbPairingId(await importDbModel(path.join(archivedir(), archiveFolder)));
+    // const currentModel = generateDbPairingId(await this.structure({ conid, database }));
+    // const currentModelPaired = matchPairedObjects(deployedModel, currentModel);
+    // const connection = await connections.get({ conid });
+    // const driver = requireEngineDriver(connection);
+    // const { sql } = getAlterDatabaseScript(currentModelPaired, deployedModel, {}, deployedModel, driver);
+    // return {
+    //   deployedModel,
+    //   currentModel,
+    //   currentModelPaired,
+    //   sql,
+    // };
+    // return sql;
+  },
+  // runCommand_meta: true,
   // async runCommand({ conid, database, sql }) {
   //   console.log(`Running SQL command , conid=${conid}, database=${database}, sql=${sql}`);
   //   const opened = await this.ensureOpened(conid, database);
   //   const res = await this.sendRequest(opened, { msgtype: 'queryData', sql });
   //   return res;
   // },
+
+  async getUnifiedDiff({ sourceConid, sourceDatabase, targetConid, targetDatabase }) {
+    const dbDiffOptions = sourceConid == '__model' ? modelCompareDbDiffOptions : {};
+
+    const sourceDb = generateDbPairingId(
+      extendDatabaseInfo(await this.structure({ conid: sourceConid, database: sourceDatabase }))
+    );
+    const targetDb = generateDbPairingId(
+      extendDatabaseInfo(await this.structure({ conid: targetConid, database: targetDatabase }))
+    );
+    // const sourceConnection = await connections.getCore({conid:sourceConid})
+    const connection = await connections.getCore({ conid: targetConid });
+    const driver = requireEngineDriver(connection);
+    const targetDbPaired = matchPairedObjects(sourceDb, targetDb, dbDiffOptions);
+    const diffRows = computeDbDiffRows(sourceDb, targetDbPaired, dbDiffOptions, driver);
+
+    // console.log('sourceDb', sourceDb);
+    // console.log('targetDb', targetDb);
+    // console.log('sourceConid, sourceDatabase', sourceConid, sourceDatabase);
+
+    let res = '';
+    for (const row of diffRows) {
+      // console.log('PAIR', row.source && row.source.pureName, row.target && row.target.pureName);
+      const unifiedDiff = createTwoFilesPatch(
+        (row.target && row.target.pureName) || '',
+        (row.source && row.source.pureName) || '',
+        getCreateObjectScript(row.target, driver),
+        getCreateObjectScript(row.source, driver),
+        '',
+        ''
+      );
+      res += unifiedDiff;
+    }
+    return res;
+  },
+
+  generateDbDiffReport_meta: true,
+  async generateDbDiffReport({ filePath, sourceConid, sourceDatabase, targetConid, targetDatabase }) {
+    const unifiedDiff = await this.getUnifiedDiff({ sourceConid, sourceDatabase, targetConid, targetDatabase });
+
+    const diffJson = parse(unifiedDiff);
+    // $: diffHtml = html(diffJson, { outputFormat: 'side-by-side', drawFileList: false });
+    const diffHtml = html(diffJson, { outputFormat: 'side-by-side' });
+
+    await fs.writeFile(filePath, diff2htmlPage(diffHtml));
+
+    return true;
+  },
 };

packages/api/src/controllers/files.js

@@ -1,9 +1,14 @@
+const uuidv1 = require('uuid/v1');
 const fs = require('fs-extra');
 const path = require('path');
-const { filesdir } = require('../utility/directories');
-const hasPermission = require('../utility/hasPermission');
+const { filesdir, archivedir, resolveArchiveFolder, uploadsdir, appdir } = require('../utility/directories');
+const getChartExport = require('../utility/getChartExport');
+const { hasPermission } = require('../utility/hasPermission');
 const socket = require('../utility/socket');
 const scheduler = require('./scheduler');
+const getDiagramExport = require('../utility/getDiagramExport');
+const apps = require('./apps');
+const getMapExport = require('../utility/getMapExport');
 
 function serialize(format, data) {
   if (format == 'text') return data;
@@ -18,21 +23,21 @@ function deserialize(format, text) {
 }
 
 module.exports = {
-  list_meta: 'get',
-  async list({ folder }) {
-    if (!hasPermission(`files/${folder}/read`)) return [];
+  list_meta: true,
+  async list({ folder }, req) {
+    if (!hasPermission(`files/${folder}/read`, req)) return [];
     const dir = path.join(filesdir(), folder);
     if (!(await fs.exists(dir))) return [];
     const files = (await fs.readdir(dir)).map(file => ({ folder, file }));
     return files;
   },
 
-  listAll_meta: 'get',
-  async listAll() {
+  listAll_meta: true,
+  async listAll(_params, req) {
     const folders = await fs.readdir(filesdir());
     const res = [];
     for (const folder of folders) {
-      if (!hasPermission(`files/${folder}/read`)) continue;
+      if (!hasPermission(`files/${folder}/read`, req)) continue;
       const dir = path.join(filesdir(), folder);
       const files = (await fs.readdir(dir)).map(file => ({ folder, file }));
       res.push(...files);
@@ -40,52 +45,107 @@ module.exports = {
     return res;
   },
 
-  delete_meta: 'post',
-  async delete({ folder, file }) {
-    if (!hasPermission(`files/${folder}/write`)) return;
+  delete_meta: true,
+  async delete({ folder, file }, req) {
+    if (!hasPermission(`files/${folder}/write`, req)) return false;
     await fs.unlink(path.join(filesdir(), folder, file));
-    socket.emitChanged(`files-changed-${folder}`);
+    socket.emitChanged(`files-changed`, { folder });
     socket.emitChanged(`all-files-changed`);
+    return true;
   },
 
-  rename_meta: 'post',
-  async rename({ folder, file, newFile }) {
-    if (!hasPermission(`files/${folder}/write`)) return;
+  rename_meta: true,
+  async rename({ folder, file, newFile }, req) {
+    if (!hasPermission(`files/${folder}/write`, req)) return false;
     await fs.rename(path.join(filesdir(), folder, file), path.join(filesdir(), folder, newFile));
-    socket.emitChanged(`files-changed-${folder}`);
+    socket.emitChanged(`files-changed`, { folder });
     socket.emitChanged(`all-files-changed`);
+    return true;
   },
 
-  load_meta: 'post',
-  async load({ folder, file, format }) {
-    if (!hasPermission(`files/${folder}/read`)) return null;
-    const text = await fs.readFile(path.join(filesdir(), folder, file), { encoding: 'utf-8' });
+  refresh_meta: true,
+  async refresh({ folders }, req) {
+    for (const folder of folders) {
+      socket.emitChanged(`files-changed`, { folder });
+      socket.emitChanged(`all-files-changed`);
+    }
+    return true;
+  },
+
+  copy_meta: true,
+  async copy({ folder, file, newFile }, req) {
+    if (!hasPermission(`files/${folder}/write`, req)) return false;
+    await fs.copyFile(path.join(filesdir(), folder, file), path.join(filesdir(), folder, newFile));
+    socket.emitChanged(`files-changed`, { folder });
+    socket.emitChanged(`all-files-changed`);
+    return true;
+  },
+
+  load_meta: true,
+  async load({ folder, file, format }, req) {
+    if (folder.startsWith('archive:')) {
+      const text = await fs.readFile(path.join(resolveArchiveFolder(folder.substring('archive:'.length)), file), {
+        encoding: 'utf-8',
+      });
+      return deserialize(format, text);
+    } else if (folder.startsWith('app:')) {
+      const text = await fs.readFile(path.join(appdir(), folder.substring('app:'.length), file), {
+        encoding: 'utf-8',
+      });
+      return deserialize(format, text);
+    } else {
+      if (!hasPermission(`files/${folder}/read`, req)) return null;
+      const text = await fs.readFile(path.join(filesdir(), folder, file), { encoding: 'utf-8' });
+      return deserialize(format, text);
+    }
+  },
+
+  loadFrom_meta: true,
+  async loadFrom({ filePath, format }, req) {
+    const text = await fs.readFile(filePath, { encoding: 'utf-8' });
     return deserialize(format, text);
   },
 
-  save_meta: 'post',
-  async save({ folder, file, data, format }) {
-    if (!hasPermission(`files/${folder}/write`)) return;
-    const dir = path.join(filesdir(), folder);
-    if (!(await fs.exists(dir))) {
-      await fs.mkdir(dir);
-    }
-    await fs.writeFile(path.join(dir, file), serialize(format, data));
-    socket.emitChanged(`files-changed-${folder}`);
-    socket.emitChanged(`all-files-changed`);
-    if (folder == 'shell') {
-      scheduler.reload();
+  save_meta: true,
+  async save({ folder, file, data, format }, req) {
+    if (folder.startsWith('archive:')) {
+      if (!hasPermission(`archive/write`, req)) return false;
+      const dir = resolveArchiveFolder(folder.substring('archive:'.length));
+      await fs.writeFile(path.join(dir, file), serialize(format, data));
+      socket.emitChanged(`archive-files-changed`, { folder: folder.substring('archive:'.length) });
+      return true;
+    } else if (folder.startsWith('app:')) {
+      if (!hasPermission(`apps/write`, req)) return false;
+      const app = folder.substring('app:'.length);
+      await fs.writeFile(path.join(appdir(), app, file), serialize(format, data));
+      socket.emitChanged(`app-files-changed`, { app });
+      socket.emitChanged('used-apps-changed');
+      apps.emitChangedDbApp(folder);
+      return true;
+    } else {
+      if (!hasPermission(`files/${folder}/write`, req)) return false;
+      const dir = path.join(filesdir(), folder);
+      if (!(await fs.exists(dir))) {
+        await fs.mkdir(dir);
+      }
+      await fs.writeFile(path.join(dir, file), serialize(format, data));
+      socket.emitChanged(`files-changed`, { folder });
+      socket.emitChanged(`all-files-changed`);
+      if (folder == 'shell') {
+        scheduler.reload();
+      }
+      return true;
     }
   },
 
-  saveAs_meta: 'post',
+  saveAs_meta: true,
   async saveAs({ filePath, data, format }) {
     await fs.writeFile(filePath, serialize(format, data));
   },
 
-  favorites_meta: 'get',
-  async favorites() {
-    if (!hasPermission(`files/favorites/read`)) return [];
+  favorites_meta: true,
+  async favorites(_params, req) {
+    if (!hasPermission(`files/favorites/read`, req)) return [];
     const dir = path.join(filesdir(), 'favorites');
     if (!(await fs.exists(dir))) return [];
     const files = await fs.readdir(dir);
@@ -101,4 +161,62 @@ module.exports = {
     }
     return res;
   },
+
+  generateUploadsFile_meta: true,
+  async generateUploadsFile({ extension }) {
+    const fileName = `${uuidv1()}.${extension || 'html'}`;
+    return {
+      fileName,
+      filePath: path.join(uploadsdir(), fileName),
+    };
+  },
+
+  exportChart_meta: true,
+  async exportChart({ filePath, title, config, image }) {
+    const fileName = path.parse(filePath).base;
+    const imageFile = fileName.replace('.html', '-preview.png');
+    const html = getChartExport(title, config, imageFile);
+    await fs.writeFile(filePath, html);
+    if (image) {
+      const index = image.indexOf('base64,');
+      if (index > 0) {
+        const data = image.substr(index + 'base64,'.length);
+        const buf = Buffer.from(data, 'base64');
+        await fs.writeFile(filePath.replace('.html', '-preview.png'), buf);
+      }
+    }
+    return true;
+  },
+
+  exportMap_meta: true,
+  async exportMap({ filePath, geoJson }) {
+    await fs.writeFile(filePath, getMapExport(geoJson));
+    return true;
+  },
+
+  exportDiagram_meta: true,
+  async exportDiagram({ filePath, html, css, themeType, themeClassName }) {
+    await fs.writeFile(filePath, getDiagramExport(html, css, themeType, themeClassName));
+    return true;
+  },
+
+  getFileRealPath_meta: true,
+  async getFileRealPath({ folder, file }, req) {
+    if (folder.startsWith('archive:')) {
+      if (!hasPermission(`archive/write`, req)) return false;
+      const dir = resolveArchiveFolder(folder.substring('archive:'.length));
+      return path.join(dir, file);
+    } else if (folder.startsWith('app:')) {
+      if (!hasPermission(`apps/write`, req)) return false;
+      const app = folder.substring('app:'.length);
+      return path.join(appdir(), app, file);
+    } else {
+      if (!hasPermission(`files/${folder}/write`, req)) return false;
+      const dir = path.join(filesdir(), folder);
+      if (!(await fs.exists(dir))) {
+        await fs.mkdir(dir);
+      }
+      return path.join(dir, file);
+    }
+  },
 };

packages/api/src/controllers/jsldata.js

@@ -1,16 +1,21 @@
+const { filterName } = require('dbgate-tools');
 const fs = require('fs');
 const lineReader = require('line-reader');
 const _ = require('lodash');
+const { __ } = require('lodash/fp');
 const DatastoreProxy = require('../utility/DatastoreProxy');
-const { saveFreeTableData } = require('../utility/freeTableStorage');
 const getJslFileName = require('../utility/getJslFileName');
 const JsonLinesDatastore = require('../utility/JsonLinesDatastore');
+const requirePluginFunction = require('../utility/requirePluginFunction');
 const socket = require('../utility/socket');
 
 function readFirstLine(file) {
   return new Promise((resolve, reject) => {
     lineReader.open(file, (err, reader) => {
-      if (err) reject(err);
+      if (err) {
+        reject(err);
+        return;
+      }
       if (reader.hasNextLine()) {
         reader.nextLine((err, line) => {
           if (err) reject(err);
@@ -94,39 +99,88 @@ module.exports = {
   //   return readerInfo;
   // },
 
-  async ensureDatastore(jslid) {
+  async ensureDatastore(jslid, formatterFunction) {
     let datastore = this.datastores[jslid];
-    if (!datastore) {
-      datastore = new JsonLinesDatastore(getJslFileName(jslid));
+    if (!datastore || datastore.formatterFunction != formatterFunction) {
+      if (datastore) {
+        datastore._closeReader();
+      }
+      datastore = new JsonLinesDatastore(getJslFileName(jslid), formatterFunction);
       // datastore = new DatastoreProxy(getJslFileName(jslid));
       this.datastores[jslid] = datastore;
     }
     return datastore;
   },
 
-  getInfo_meta: 'get',
+  async closeDataStore(jslid) {
+    const datastore = this.datastores[jslid];
+    if (datastore) {
+      await datastore._closeReader();
+      delete this.datastores[jslid];
+    }
+  },
+
+  getInfo_meta: true,
   async getInfo({ jslid }) {
     const file = getJslFileName(jslid);
-    const firstLine = await readFirstLine(file);
-    if (firstLine) return JSON.parse(firstLine);
-    return null;
+    try {
+      const firstLine = await readFirstLine(file);
+      if (firstLine) {
+        const parsed = JSON.parse(firstLine);
+        if (parsed.__isStreamHeader) {
+          return parsed;
+        }
+        return {
+          __isStreamHeader: true,
+          __isDynamicStructure: true,
+        };
+      }
+      return null;
+    } catch (err) {
+      return null;
+    }
   },
 
-  getRows_meta: 'post',
-  async getRows({ jslid, offset, limit, filters }) {
-    const datastore = await this.ensureDatastore(jslid);
-    return datastore.getRows(offset, limit, _.isEmpty(filters) ? null : filters);
+  getRows_meta: true,
+  async getRows({ jslid, offset, limit, filters, sort, formatterFunction }) {
+    const datastore = await this.ensureDatastore(jslid, formatterFunction);
+    return datastore.getRows(offset, limit, _.isEmpty(filters) ? null : filters, _.isEmpty(sort) ? null : sort);
   },
 
-  getStats_meta: 'get',
+  exists_meta: true,
+  async exists({ jslid }) {
+    const fileName = getJslFileName(jslid);
+    return fs.existsSync(fileName);
+  },
+
+  getStats_meta: true,
   getStats({ jslid }) {
     const file = `${getJslFileName(jslid)}.stats`;
-    if (fs.existsSync(file)) return JSON.parse(fs.readFileSync(file, 'utf-8'));
+    if (fs.existsSync(file)) {
+      try {
+        return JSON.parse(fs.readFileSync(file, 'utf-8'));
+      } catch (e) {
+        return {};
+      }
+    }
     return {};
   },
 
+  loadFieldValues_meta: true,
+  async loadFieldValues({ jslid, field, search, formatterFunction }) {
+    const datastore = await this.ensureDatastore(jslid, formatterFunction);
+    const res = new Set();
+    await datastore.enumRows(row => {
+      if (!filterName(search, row[field])) return true;
+      res.add(row[field]);
+      return res.size < 100;
+    });
+    // @ts-ignore
+    return [...res].map(value => ({ value }));
+  },
+
   async notifyChangedStats(stats) {
-    console.log('SENDING STATS', JSON.stringify(stats));
+    // console.log('SENDING STATS', JSON.stringify(stats));
     const datastore = this.datastores[stats.jslid];
     if (datastore) await datastore.notifyChanged();
     socket.emit(`jsldata-stats-${stats.jslid}`, stats);
@@ -140,9 +194,100 @@ module.exports = {
     // }
   },
 
-  saveFreeTable_meta: 'post',
-  async saveFreeTable({ jslid, data }) {
-    saveFreeTableData(getJslFileName(jslid), data);
+  saveText_meta: true,
+  async saveText({ jslid, text }) {
+    await fs.promises.writeFile(getJslFileName(jslid), text);
     return true;
   },
+
+  saveRows_meta: true,
+  async saveRows({ jslid, rows }) {
+    const fileStream = fs.createWriteStream(getJslFileName(jslid));
+    for (const row of rows) {
+      await fileStream.write(JSON.stringify(row) + '\n');
+    }
+    await fileStream.close();
+    return true;
+  },
+
+  extractTimelineChart_meta: true,
+  async extractTimelineChart({ jslid, timestampFunction, aggregateFunction, measures }) {
+    const timestamp = requirePluginFunction(timestampFunction);
+    const aggregate = requirePluginFunction(aggregateFunction);
+    const datastore = new JsonLinesDatastore(getJslFileName(jslid));
+    let mints = null;
+    let maxts = null;
+    // pass 1 - counts stats, time range
+    await datastore.enumRows(row => {
+      const ts = timestamp(row);
+      if (!mints || ts < mints) mints = ts;
+      if (!maxts || ts > maxts) maxts = ts;
+      return true;
+    });
+    const minTime = new Date(mints).getTime();
+    const maxTime = new Date(maxts).getTime();
+    const duration = maxTime - minTime;
+    const STEPS = 100;
+    let stepCount = duration > 100 * 1000 ? STEPS : Math.round((maxTime - minTime) / 1000);
+    if (stepCount < 2) {
+      stepCount = 2;
+    }
+    const stepDuration = duration / stepCount;
+    const labels = _.range(stepCount).map(i => new Date(minTime + stepDuration / 2 + stepDuration * i));
+
+    // const datasets = measures.map(m => ({
+    //   label: m.label,
+    //   data: Array(stepCount).fill(0),
+    // }));
+
+    const mproc = measures.map(m => ({
+      ...m,
+    }));
+
+    const data = Array(stepCount)
+      .fill(0)
+      .map(() => ({}));
+
+    // pass 2 - count measures
+    await datastore.enumRows(row => {
+      const ts = timestamp(row);
+      let part = Math.round((new Date(ts).getTime() - minTime) / stepDuration);
+      if (part < 0) part = 0;
+      if (part >= stepCount) part - stepCount - 1;
+      if (data[part]) {
+        data[part] = aggregate(data[part], row, stepDuration);
+      }
+      return true;
+    });
+
+    datastore._closeReader();
+
+    // const measureByField = _.fromPairs(measures.map((m, i) => [m.field, i]));
+
+    // for (let mindex = 0; mindex < measures.length; mindex++) {
+    //   for (let stepIndex = 0; stepIndex < stepCount; stepIndex++) {
+    //     const measure = measures[mindex];
+    //     if (measure.perSecond) {
+    //       datasets[mindex].data[stepIndex] /= stepDuration / 1000;
+    //     }
+    //     if (measure.perField) {
+    //       datasets[mindex].data[stepIndex] /= datasets[measureByField[measure.perField]].data[stepIndex];
+    //     }
+    //   }
+    // }
+
+    // for (let i = 0; i < measures.length; i++) {
+    //   if (measures[i].hidden) {
+    //     datasets[i] = null;
+    //   }
+    // }
+
+    return {
+      labels,
+      datasets: mproc.map(m => ({
+        label: m.label,
+        data: data.map(d => d[m.field] || 0),
+      })),
+    };
+  },
 };

packages/api/src/controllers/metadata.js

@@ -7,17 +7,17 @@ function pickObjectNames(array) {
 }
 
 module.exports = {
-  // tableData_meta: 'get',
+  // tableData_meta: true,
   // async tableData({ conid, database, schemaName, pureName }) {
   //   const opened = await databaseConnections.ensureOpened(conid, database);
   //   const res = await databaseConnections.sendRequest(opened, { msgtype: 'tableData', schemaName, pureName });
   //   return res;
   // },
 
-  listObjects_meta: 'get',
+  listObjects_meta: true,
   async listObjects({ conid, database }) {
     const opened = await databaseConnections.ensureOpened(conid, database);
-    const types = ['tables', 'views', 'procedures', 'functions', 'triggers'];
+    const types = ['tables', 'collections', 'views', 'procedures', 'functions', 'triggers'];
     return types.reduce(
       (res, type) => ({
         ...res,
@@ -27,7 +27,7 @@ module.exports = {
     );
   },
 
-  tableInfo_meta: 'get',
+  tableInfo_meta: true,
   async tableInfo({ conid, database, schemaName, pureName }) {
     const opened = await databaseConnections.ensureOpened(conid, database);
     const table = opened.structure.tables.find(x => x.pureName == pureName && x.schemaName == schemaName);
@@ -38,7 +38,7 @@ module.exports = {
     };
   },
 
-  sqlObjectInfo_meta: 'get',
+  sqlObjectInfo_meta: true,
   async sqlObjectInfo({ objectTypeField, conid, database, schemaName, pureName }) {
     const opened = await databaseConnections.ensureOpened(conid, database);
     const res = opened.structure[objectTypeField].find(x => x.pureName == pureName && x.schemaName == schemaName);

packages/api/src/controllers/plugins.js

@@ -2,50 +2,35 @@ const fs = require('fs-extra');
 const axios = require('axios');
 const path = require('path');
 const { extractPackageName } = require('dbgate-tools');
-const { pluginsdir, datadir } = require('../utility/directories');
+const { pluginsdir, packagedPluginsDir } = require('../utility/directories');
 const socket = require('../utility/socket');
 const compareVersions = require('compare-versions');
 const requirePlugin = require('../shell/requirePlugin');
 const downloadPackage = require('../utility/downloadPackage');
-const hasPermission = require('../utility/hasPermission');
-
-// async function loadPackageInfo(dir) {
-//   const readmeFile = path.join(dir, 'README.md');
-//   const packageFile = path.join(dir, 'package.json');
-
-//   if (!(await fs.exists(packageFile))) {
-//     return null;
-//   }
-
-//   let readme = null;
-//   let manifest = null;
-//   if (await fs.exists(readmeFile)) readme = await fs.readFile(readmeFile, { encoding: 'utf-8' });
-//   if (await fs.exists(packageFile)) manifest = JSON.parse(await fs.readFile(packageFile, { encoding: 'utf-8' }));
-//   return {
-//     readme,
-//     manifest,
-//   };
-// }
-
-const preinstallPluginMinimalVersions = {
-  'dbgate-plugin-mssql': '1.1.0',
-  'dbgate-plugin-mysql': '1.1.0',
-  'dbgate-plugin-postgres': '1.1.0',
-  'dbgate-plugin-csv': '1.0.8',
-  'dbgate-plugin-excel': '1.0.6',
-};
+const { hasPermission } = require('../utility/hasPermission');
+const _ = require('lodash');
+const packagedPluginsContent = require('../packagedPluginsContent');
 
 module.exports = {
-  script_meta: 'get',
+  script_meta: true,
   async script({ packageName }) {
-    const file = path.join(pluginsdir(), packageName, 'dist', 'frontend.js');
+    const packagedContent = packagedPluginsContent();
+
+    if (packagedContent && packagedContent[packageName]) {
+      return packagedContent[packageName].frontend;
+    }
+
+    const file1 = path.join(packagedPluginsDir(), packageName, 'dist', 'frontend.js');
+    const file2 = path.join(pluginsdir(), packageName, 'dist', 'frontend.js');
+    // @ts-ignore
+    const file = (await fs.exists(file1)) ? file1 : file2;
     const data = await fs.readFile(file, {
       encoding: 'utf-8',
     });
     return data;
   },
 
-  search_meta: 'get',
+  search_meta: true,
   async search({ filter }) {
     // DOCS: https://github.com/npm/registry/blob/master/docs/REGISTRY-API.md#get-v1search
     const resp = await axios.default.get(
@@ -55,17 +40,20 @@ module.exports = {
     return (objects || []).map(x => x.package);
   },
 
-  info_meta: 'get',
+  info_meta: true,
   async info({ packageName }) {
     try {
       const infoResp = await axios.default.get(`https://registry.npmjs.org/${packageName}`);
       const { latest } = infoResp.data['dist-tags'];
       const manifest = infoResp.data.versions[latest];
       const { readme } = infoResp.data;
+      // @ts-ignore
+      const isPackaged = await fs.exists(path.join(packagedPluginsDir(), packageName));
 
       return {
         readme,
         manifest,
+        isPackaged,
       };
     } catch (err) {
       return {
@@ -73,130 +61,154 @@ module.exports = {
         error: err.message,
       };
     }
-
-    // const dir = path.join(pluginstmpdir(), packageName);
-    // if (!(await fs.exists(dir))) {
-    //   await downloadPackage(packageName, dir);
-    // }
-    // return await loadPackageInfo(dir);
-    // return await {
-    //   ...loadPackageInfo(dir),
-    //   installed: loadPackageInfo(path.join(pluginsdir(), packageName)),
-    // };
   },
 
-  installed_meta: 'get',
+  installed_meta: true,
   async installed() {
-    const files = await fs.readdir(pluginsdir());
+    const packagedContent = packagedPluginsContent();
+
+    const files1 = packagedContent ? _.keys(packagedContent) : await fs.readdir(packagedPluginsDir());
+    const files2 = await fs.readdir(pluginsdir());
+
     const res = [];
-    for (const packageName of files) {
-      const manifest = await fs.readFile(path.join(pluginsdir(), packageName, 'package.json')).then(x => JSON.parse(x));
-      const readmeFile = path.join(pluginsdir(), packageName, 'README.md');
-      if (await fs.exists(readmeFile)) {
-        manifest.readme = await fs.readFile(readmeFile, { encoding: 'utf-8' });
+    for (const packageName of _.union(files1, files2)) {
+      if (packageName == 'dist') continue;
+      if (!/^dbgate-plugin-.*$/.test(packageName)) continue;
+      try {
+        if (packagedContent && packagedContent[packageName]) {
+          const manifest = {
+            ...packagedContent[packageName].manifest,
+          };
+          manifest.isPackaged = true;
+          manifest.readme = packagedContent[packageName].readme;
+          res.push(manifest);
+        } else {
+          const isPackaged = files1.includes(packageName);
+          const manifest = await fs
+            .readFile(path.join(isPackaged ? packagedPluginsDir() : pluginsdir(), packageName, 'package.json'), {
+              encoding: 'utf-8',
+            })
+            .then(x => JSON.parse(x));
+          if (!manifest.keywords) {
+            continue;
+          }
+          if (!manifest.keywords.includes('dbgateplugin')) {
+            continue;
+          }
+          const readmeFile = path.join(isPackaged ? packagedPluginsDir() : pluginsdir(), packageName, 'README.md');
+          // @ts-ignore
+          if (await fs.exists(readmeFile)) {
+            manifest.readme = await fs.readFile(readmeFile, { encoding: 'utf-8' });
+          }
+          manifest.isPackaged = isPackaged;
+          res.push(manifest);
+        }
+      } catch (err) {
+        console.log(`Skipped plugin ${packageName}, error:`, err.message);
       }
-      res.push(manifest);
     }
     return res;
-    // const res = await Promise.all(
-    //   files.map((packageName) =>
-    //     fs.readFile(path.join(pluginsdir(), packageName, 'package.json')).then((x) => JSON.parse(x))
-    //   )
-    // );
   },
 
-  async saveRemovePlugins() {
-    await fs.writeFile(path.join(datadir(), 'removed-plugins'), this.removedPlugins.join('\n'));
-  },
+  // async saveRemovePlugins() {
+  //   await fs.writeFile(path.join(datadir(), 'removed-plugins'), this.removedPlugins.join('\n'));
+  // },
 
-  install_meta: 'post',
-  async install({ packageName }) {
-    if (!hasPermission(`plugins/install`)) return;
+  install_meta: true,
+  async install({ packageName }, req) {
+    if (!hasPermission(`plugins/install`, req)) return;
     const dir = path.join(pluginsdir(), packageName);
+    // @ts-ignore
     if (!(await fs.exists(dir))) {
       await downloadPackage(packageName, dir);
     }
     socket.emitChanged(`installed-plugins-changed`);
-    this.removedPlugins = this.removedPlugins.filter(x => x != packageName);
-    await this.saveRemovePlugins();
+    // this.removedPlugins = this.removedPlugins.filter(x => x != packageName);
+    // await this.saveRemovePlugins();
+    return true;
   },
 
-  uninstall_meta: 'post',
-  async uninstall({ packageName }) {
-    if (!hasPermission(`plugins/install`)) return;
+  uninstall_meta: true,
+  async uninstall({ packageName }, req) {
+    if (!hasPermission(`plugins/install`, req)) return;
     const dir = path.join(pluginsdir(), packageName);
     await fs.rmdir(dir, { recursive: true });
     socket.emitChanged(`installed-plugins-changed`);
-    this.removedPlugins.push(packageName);
-    await this.saveRemovePlugins();
+    // this.removedPlugins.push(packageName);
+    // await this.saveRemovePlugins();
+    return true;
   },
 
-  upgrade_meta: 'post',
-  async upgrade({ packageName }) {
-    if (!hasPermission(`plugins/install`)) return;
+  upgrade_meta: true,
+  async upgrade({ packageName }, req) {
+    if (!hasPermission(`plugins/install`, req)) return;
     const dir = path.join(pluginsdir(), packageName);
+    // @ts-ignore
     if (await fs.exists(dir)) {
       await fs.rmdir(dir, { recursive: true });
       await downloadPackage(packageName, dir);
     }
 
     socket.emitChanged(`installed-plugins-changed`);
+    return true;
   },
 
-  command_meta: 'post',
+  command_meta: true,
   async command({ packageName, command, args }) {
     const content = requirePlugin(packageName);
     return content.commands[command](args);
   },
 
-  authTypes_meta: 'post',
+  authTypes_meta: true,
   async authTypes({ engine }) {
     const packageName = extractPackageName(engine);
+    if (!packageName) return null;
     const content = requirePlugin(packageName);
-    if (!content.driver || content.driver.engine != engine || !content.driver.getAuthTypes) return null;
-    return content.driver.getAuthTypes() || null;
+    const driver = content.drivers.find(x => x.engine == engine);
+    if (!driver || !driver.getAuthTypes) return null;
+    return driver.getAuthTypes() || null;
   },
 
-  async _init() {
-    const installed = await this.installed();
-    try {
-      this.removedPlugins = (await fs.readFile(path.join(datadir(), 'removed-plugins'), { encoding: 'utf-8' })).split(
-        '\n'
-      );
-    } catch (err) {
-      this.removedPlugins = [];
-    }
+  // async _init() {
+  //   const installed = await this.installed();
+  //   try {
+  //     this.removedPlugins = (await fs.readFile(path.join(datadir(), 'removed-plugins'), { encoding: 'utf-8' })).split(
+  //       '\n'
+  //     );
+  //   } catch (err) {
+  //     this.removedPlugins = [];
+  //   }
 
-    for (const packageName of Object.keys(preinstallPluginMinimalVersions)) {
-      const installedVersion = installed.find(x => x.name == packageName);
-      if (installedVersion) {
-        // plugin installed, test, whether upgrade
-        const requiredVersion = preinstallPluginMinimalVersions[packageName];
-        if (compareVersions(installedVersion.version, requiredVersion) < 0) {
-          console.log(
-            `Upgrading preinstalled plugin ${packageName}, found ${installedVersion.version}, required version ${requiredVersion}`
-          );
-          await this.upgrade({ packageName });
-        } else {
-          console.log(
-            `Plugin ${packageName} not upgraded, found ${installedVersion.version}, required version ${requiredVersion}`
-          );
-        }
+  //   for (const packageName of Object.keys(preinstallPluginMinimalVersions)) {
+  //     const installedVersion = installed.find(x => x.name == packageName);
+  //     if (installedVersion) {
+  //       // plugin installed, test, whether upgrade
+  //       const requiredVersion = preinstallPluginMinimalVersions[packageName];
+  //       if (compareVersions(installedVersion.version, requiredVersion) < 0) {
+  //         console.log(
+  //           `Upgrading preinstalled plugin ${packageName}, found ${installedVersion.version}, required version ${requiredVersion}`
+  //         );
+  //         await this.upgrade({ packageName });
+  //       } else {
+  //         console.log(
+  //           `Plugin ${packageName} not upgraded, found ${installedVersion.version}, required version ${requiredVersion}`
+  //         );
+  //       }
 
-        continue;
-      }
+  //       continue;
+  //     }
 
-      if (this.removedPlugins.includes(packageName)) {
-        // plugin was remvoed, don't install again
-        continue;
-      }
+  //     if (this.removedPlugins.includes(packageName)) {
+  //       // plugin was remvoed, don't install again
+  //       continue;
+  //     }
 
-      try {
-        console.log('Preinstalling plugin', packageName);
-        await this.install({ packageName });
-      } catch (err) {
-        console.error('Error preinstalling plugin', packageName, err);
-      }
-    }
-  },
+  //     try {
+  //       console.log('Preinstalling plugin', packageName);
+  //       await this.install({ packageName });
+  //     } catch (err) {
+  //       console.error('Error preinstalling plugin', packageName, err);
+  //     }
+  //   }
+  // },
 };

packages/api/src/controllers/queryHistory.js

@@ -0,0 +1,54 @@
+const fsReverse = require('fs-reverse');
+const fs = require('fs-extra');
+const path = require('path');
+const { datadir } = require('../utility/directories');
+const _ = require('lodash');
+const { filterName } = require('dbgate-tools');
+const socket = require('../utility/socket');
+
+function readCore(reader, skip, limit, filter) {
+  return new Promise((resolve, reject) => {
+    const res = [];
+    let readed = 0;
+    reader.on('data', line => {
+      if (!line && !line.trim()) return;
+      try {
+        const json = JSON.parse(line);
+        if (filterName(filter, json.sql, json.database)) {
+          if (!skip || readed >= skip) {
+            res.push(json);
+          }
+          readed++;
+          if (limit && readed > (skip || 0) + limit) {
+            reader.destroy();
+            resolve(res);
+          }
+        }
+      } catch (err) {
+        reader.destroy();
+        reject(err);
+      }
+    });
+    reader.on('end', () => resolve(res));
+  });
+}
+
+module.exports = {
+  read_meta: true,
+  async read({ skip, limit, filter }) {
+    const fileName = path.join(datadir(), 'query-history.jsonl');
+    // @ts-ignore
+    if (!(await fs.exists(fileName))) return [];
+    const reader = fsReverse(fileName);
+    const res = await readCore(reader, skip, limit, filter);
+    return res;
+  },
+
+  write_meta: true,
+  async write({ data }) {
+    const fileName = path.join(datadir(), 'query-history.jsonl');
+    await fs.appendFile(fileName, JSON.stringify(data) + '\n');
+    socket.emit('query-history-changed');
+    return 'OK';
+  },
+};

packages/api/src/controllers/runners.js

@@ -5,9 +5,18 @@ const uuidv1 = require('uuid/v1');
 const byline = require('byline');
 const socket = require('../utility/socket');
 const { fork } = require('child_process');
-const { rundir, uploadsdir, pluginsdir } = require('../utility/directories');
-const { extractShellApiPlugins, extractShellApiFunctionName } = require('dbgate-tools');
+const { rundir, uploadsdir, pluginsdir, getPluginBackendPath, packagedPluginList } = require('../utility/directories');
+const {
+  extractShellApiPlugins,
+  extractShellApiFunctionName,
+  jsonScriptToJavascript,
+  getLogger,
+  safeJsonParse,
+} = require('dbgate-tools');
 const { handleProcessCommunication } = require('../utility/processComm');
+const processArgs = require('../utility/processArgs');
+const platformInfo = require('../utility/platformInfo');
+const logger = getLogger('runners');
 
 function extractPlugins(script) {
   const requireRegex = /\s*\/\/\s*@require\s+([^\s]+)\s*\n/g;
@@ -15,22 +24,26 @@ function extractPlugins(script) {
   return matches.map(x => x[1]);
 }
 
-const requirePluginsTemplate = plugins =>
+const requirePluginsTemplate = (plugins, isExport) =>
   plugins
     .map(
-      packageName => `const ${_.camelCase(packageName)} = require(process.env.PLUGIN_${_.camelCase(packageName)});\n`
+      packageName =>
+        `const ${_.camelCase(packageName)} = require(${
+          isExport ? `'${packageName}'` : `process.env.PLUGIN_${_.camelCase(packageName)}`
+        });\n`
     )
     .join('') + `dbgateApi.registerPlugins(${plugins.map(x => _.camelCase(x)).join(',')});\n`;
 
-const scriptTemplate = script => `
-const dbgateApi = require(process.env.DBGATE_API);
+const scriptTemplate = (script, isExport) => `
+const dbgateApi = require(${isExport ? `'dbgate-api'` : 'process.env.DBGATE_API'});
+const logger = dbgateApi.getLogger('script');
 dbgateApi.initializeApiEnvironment();
-${requirePluginsTemplate(extractPlugins(script))}
+${requirePluginsTemplate(extractPlugins(script), isExport)}
 require=null;
 async function run() {
 ${script}
 await dbgateApi.finalizer.run();
-console.log('Finished job script');
+logger.info('Finished job script');
 }
 dbgateApi.runScript(run);
 `;
@@ -54,20 +67,23 @@ module.exports = {
   requests: {},
 
   dispatchMessage(runid, message) {
-    if (message) console.log('...', message.message);
-    if (_.isString(message)) {
-      socket.emit(`runner-info-${runid}`, {
-        message,
+    if (message) {
+      const json = safeJsonParse(message.message);
+
+      if (json) logger.log(json);
+      else logger.info(message.message);
+
+      const toEmit = {
         time: new Date(),
-        severity: 'info',
-      });
-    }
-    if (_.isPlainObject(message)) {
-      socket.emit(`runner-info-${runid}`, {
-        time: new Date(),
-        severity: 'info',
         ...message,
-      });
+        message: json ? json.msg : message.message,
+      };
+
+      if (json && json.level >= 50) {
+        toEmit.severity = 'error';
+      }
+
+      socket.emit(`runner-info-${runid}`, toEmit);
     }
   },
 
@@ -92,26 +108,37 @@ module.exports = {
     const scriptFile = path.join(uploadsdir(), runid + '.js');
     fs.writeFileSync(`${scriptFile}`, scriptText);
     fs.mkdirSync(directory);
-    const pluginNames = fs.readdirSync(pluginsdir());
-    console.log(`RUNNING SCRIPT ${scriptFile}`);
+    const pluginNames = _.union(fs.readdirSync(pluginsdir()), packagedPluginList);
+    logger.info({ scriptFile }, 'Running script');
     // const subprocess = fork(scriptFile, ['--checkParent', '--max-old-space-size=8192'], {
-    const subprocess = fork(scriptFile, ['--checkParent', ...process.argv.slice(3)], {
-      cwd: directory,
-      stdio: ['ignore', 'pipe', 'pipe', 'ipc'],
-      env: {
-        ...process.env,
-        DBGATE_API: global['dbgateApiModulePath'] || process.argv[1],
-        ..._.fromPairs(pluginNames.map(name => [`PLUGIN_${_.camelCase(name)}`, path.join(pluginsdir(), name)])),
-      },
-    });
-    const pipeDispatcher = severity => data =>
-      this.dispatchMessage(runid, { severity, message: data.toString().trim() });
+    const subprocess = fork(
+      scriptFile,
+      [
+        '--checkParent', // ...process.argv.slice(3)
+        '--is-forked-api',
+        '--process-display-name',
+        'script',
+        ...processArgs.getPassArgs(),
+      ],
+      {
+        cwd: directory,
+        stdio: ['ignore', 'pipe', 'pipe', 'ipc'],
+        env: {
+          ...process.env,
+          DBGATE_API: global['API_PACKAGE'] || process.argv[1],
+          ..._.fromPairs(pluginNames.map(name => [`PLUGIN_${_.camelCase(name)}`, getPluginBackendPath(name)])),
+        },
+      }
+    );
+    const pipeDispatcher = severity => data => {
+      return this.dispatchMessage(runid, { severity, message: data.toString().trim() });
+    };
 
     byline(subprocess.stdout).on('data', pipeDispatcher('info'));
     byline(subprocess.stderr).on('data', pipeDispatcher('error'));
     subprocess.on('exit', code => {
       this.rejectRequest(runid, { message: 'No data retured, maybe input data source is too big' });
-      console.log('... EXIT process', code);
+      logger.info({ code, pid: subprocess.pid }, 'Exited process');
       socket.emit(`runner-done-${runid}`, code);
     });
     subprocess.on('error', error => {
@@ -133,16 +160,31 @@ module.exports = {
       if (handleProcessCommunication(message, subprocess)) return;
       this[`handle_${msgtype}`](runid, message);
     });
-    return newOpened;
+    return _.pick(newOpened, ['runid']);
   },
 
-  start_meta: 'post',
+  start_meta: true,
   async start({ script }) {
     const runid = uuidv1();
-    return this.startCore(runid, scriptTemplate(script));
+
+    if (script.type == 'json') {
+      const js = jsonScriptToJavascript(script);
+      return this.startCore(runid, scriptTemplate(js, false));
+    }
+
+    if (!platformInfo.allowShellScripting) {
+      return { errorMessage: 'Shell scripting is not allowed' };
+    }
+
+    return this.startCore(runid, scriptTemplate(script, false));
   },
 
-  cancel_meta: 'post',
+  getNodeScript_meta: true,
+  async getNodeScript({ script }) {
+    return scriptTemplate(script, true);
+  },
+
+  cancel_meta: true,
   async cancel({ runid }) {
     const runner = this.opened.find(x => x.runid == runid);
     if (!runner) {
@@ -152,7 +194,7 @@ module.exports = {
     return { state: 'ok' };
   },
 
-  files_meta: 'get',
+  files_meta: true,
   async files({ runid }) {
     const directory = path.join(rundir(), runid);
     const files = await fs.readdir(directory);
@@ -168,7 +210,7 @@ module.exports = {
     return res;
   },
 
-  loadReader_meta: 'post',
+  loadReader_meta: true,
   async loadReader({ functionName, props }) {
     const promise = new Promise((resolve, reject) => {
       const runid = uuidv1();

packages/api/src/controllers/scheduler.js

@@ -3,7 +3,10 @@ const fs = require('fs-extra');
 const path = require('path');
 const cron = require('node-cron');
 const runners = require('./runners');
-const hasPermission = require('../utility/hasPermission');
+const { hasPermission } = require('../utility/hasPermission');
+const { getLogger } = require('dbgate-tools');
+
+const logger = getLogger('scheduler');
 
 const scheduleRegex = /\s*\/\/\s*@schedule\s+([^\n]+)\n/;
 
@@ -21,13 +24,13 @@ module.exports = {
     if (!match) return;
     const pattern = match[1];
     if (!cron.validate(pattern)) return;
-    console.log(`Schedule script ${file} with pattern ${pattern}`);
+    logger.info(`Schedule script ${file} with pattern ${pattern}`);
     const task = cron.schedule(pattern, () => runners.start({ script: text }));
     this.tasks.push(task);
   },
 
-  async reload() {
-    if (!hasPermission('files/shell/read')) return;
+  async reload(_params, req) {
+    if (!hasPermission('files/shell/read', req)) return;
     const shellDir = path.join(filesdir(), 'shell');
     await this.unload();
     if (!(await fs.exists(shellDir))) return;

packages/api/src/controllers/serverConnections.js

@@ -1,21 +1,37 @@
 const connections = require('./connections');
 const socket = require('../utility/socket');
 const { fork } = require('child_process');
+const uuidv1 = require('uuid/v1');
 const _ = require('lodash');
 const AsyncLock = require('async-lock');
 const { handleProcessCommunication } = require('../utility/processComm');
 const lock = new AsyncLock();
+const config = require('./config');
+const processArgs = require('../utility/processArgs');
+const { testConnectionPermission } = require('../utility/hasPermission');
+const { MissingCredentialsError } = require('../utility/exceptions');
+const pipeForkLogs = require('../utility/pipeForkLogs');
+const { getLogger } = require('dbgate-tools');
+
+const logger = getLogger('serverConnection');
 
 module.exports = {
   opened: [],
   closed: {},
   lastPinged: {},
+  requests: {},
 
   handle_databases(conid, { databases }) {
     const existing = this.opened.find(x => x.conid == conid);
     if (!existing) return;
     existing.databases = databases;
-    socket.emitChanged(`database-list-changed-${conid}`);
+    socket.emitChanged(`database-list-changed`, { conid });
+  },
+  handle_version(conid, { version }) {
+    const existing = this.opened.find(x => x.conid == conid);
+    if (!existing) return;
+    existing.version = version;
+    socket.emitChanged(`server-version-changed`, { conid });
   },
   handle_status(conid, { status }) {
     const existing = this.opened.find(x => x.conid == conid);
@@ -24,13 +40,37 @@ module.exports = {
     socket.emitChanged(`server-status-changed`);
   },
   handle_ping() {},
+  handle_response(conid, { msgid, ...response }) {
+    const [resolve, reject] = this.requests[msgid];
+    resolve(response);
+    delete this.requests[msgid];
+  },
 
   async ensureOpened(conid) {
     const res = await lock.acquire(conid, async () => {
       const existing = this.opened.find(x => x.conid == conid);
       if (existing) return existing;
-      const connection = await connections.get({ conid });
-      const subprocess = fork(process.argv[1], ['serverConnectionProcess', ...process.argv.slice(3)]);
+      const connection = await connections.getCore({ conid });
+      if (!connection) {
+        throw new Error(`Connection with conid="${conid}" not fund`);
+      }
+      if (connection.passwordMode == 'askPassword' || connection.passwordMode == 'askUser') {
+        throw new MissingCredentialsError({ conid, passwordMode: connection.passwordMode });
+      }
+      const subprocess = fork(
+        global['API_PACKAGE'] || process.argv[1],
+        [
+          '--is-forked-api',
+          '--start-process',
+          'serverConnectionProcess',
+          ...processArgs.getPassArgs(),
+          // ...process.argv.slice(3),
+        ],
+        {
+          stdio: ['ignore', 'pipe', 'pipe', 'ipc'],
+        }
+      );
+      pipeForkLogs(subprocess);
       const newOpened = {
         conid,
         subprocess,
@@ -55,7 +95,7 @@ module.exports = {
         if (newOpened.disconnected) return;
         this.close(conid, false);
       });
-      subprocess.send({ msgtype: 'connect', ...connection });
+      subprocess.send({ msgtype: 'connect', ...connection, globalSettings: await config.getSettings() });
       return newOpened;
     });
     return res;
@@ -65,7 +105,13 @@ module.exports = {
     const existing = this.opened.find(x => x.conid == conid);
     if (existing) {
       existing.disconnected = true;
-      if (kill) existing.subprocess.kill();
+      if (kill) {
+        try {
+          existing.subprocess.kill();
+        } catch (err) {
+          logger.error({ err }, 'Error killing subprocess');
+        }
+      }
       this.opened = this.opened.filter(x => x.conid != conid);
       this.closed[conid] = {
         ...existing.status,
@@ -75,13 +121,29 @@ module.exports = {
     }
   },
 
-  listDatabases_meta: 'get',
-  async listDatabases({ conid }) {
+  disconnect_meta: true,
+  async disconnect({ conid }, req) {
+    testConnectionPermission(conid, req);
+    await this.close(conid, true);
+    return { status: 'ok' };
+  },
+
+  listDatabases_meta: true,
+  async listDatabases({ conid }, req) {
+    if (!conid) return [];
+    testConnectionPermission(conid, req);
     const opened = await this.ensureOpened(conid);
     return opened.databases;
   },
 
-  serverStatus_meta: 'get',
+  version_meta: true,
+  async version({ conid }, req) {
+    testConnectionPermission(conid, req);
+    const opened = await this.ensureOpened(conid);
+    return opened.version;
+  },
+
+  serverStatus_meta: true,
   async serverStatus() {
     return {
       ...this.closed,
@@ -89,34 +151,93 @@ module.exports = {
     };
   },
 
-  ping_meta: 'post',
-  async ping({ connections }) {
+  ping_meta: true,
+  async ping({ conidArray }) {
     await Promise.all(
-      _.uniq(connections).map(async conid => {
+      _.uniq(conidArray).map(async conid => {
         const last = this.lastPinged[conid];
         if (last && new Date().getTime() - last < 30 * 1000) {
           return Promise.resolve();
         }
         this.lastPinged[conid] = new Date().getTime();
         const opened = await this.ensureOpened(conid);
-        opened.subprocess.send({ msgtype: 'ping' });
+        try {
+          opened.subprocess.send({ msgtype: 'ping' });
+        } catch (err) {
+          logger.error({ err }, 'Error pinging server connection');
+          this.close(conid);
+        }
       })
     );
     return { status: 'ok' };
   },
 
-  refresh_meta: 'post',
-  async refresh({ conid }) {
-    this.close(conid);
+  refresh_meta: true,
+  async refresh({ conid, keepOpen }, req) {
+    testConnectionPermission(conid, req);
+    if (!keepOpen) this.close(conid);
 
     await this.ensureOpened(conid);
     return { status: 'ok' };
   },
 
-  createDatabase_meta: 'post',
-  async createDatabase({ conid, name }) {
+  createDatabase_meta: true,
+  async createDatabase({ conid, name }, req) {
+    testConnectionPermission(conid, req);
     const opened = await this.ensureOpened(conid);
+    if (opened.connection.isReadOnly) return false;
     opened.subprocess.send({ msgtype: 'createDatabase', name });
     return { status: 'ok' };
   },
+
+  dropDatabase_meta: true,
+  async dropDatabase({ conid, name }, req) {
+    testConnectionPermission(conid, req);
+    const opened = await this.ensureOpened(conid);
+    if (opened.connection.isReadOnly) return false;
+    opened.subprocess.send({ msgtype: 'dropDatabase', name });
+    return { status: 'ok' };
+  },
+
+  sendRequest(conn, message) {
+    const msgid = uuidv1();
+    const promise = new Promise((resolve, reject) => {
+      this.requests[msgid] = [resolve, reject];
+      try {
+        conn.subprocess.send({ msgid, ...message });
+      } catch (err) {
+        logger.error({ err }, 'Error sending request');
+        this.close(conn.conid);
+      }
+    });
+    return promise;
+  },
+
+  async loadDataCore(msgtype, { conid, ...args }, req) {
+    testConnectionPermission(conid, req);
+    const opened = await this.ensureOpened(conid);
+    const res = await this.sendRequest(opened, { msgtype, ...args });
+    if (res.errorMessage) {
+      console.error(res.errorMessage);
+
+      return {
+        errorMessage: res.errorMessage,
+      };
+    }
+    return res.result || null;
+  },
+
+  serverSummary_meta: true,
+  async serverSummary({ conid }, req) {
+    testConnectionPermission(conid, req);
+    return this.loadDataCore('serverSummary', { conid });
+  },
+
+  summaryCommand_meta: true,
+  async summaryCommand({ conid, command, row }, req) {
+    testConnectionPermission(conid, req);
+    const opened = await this.ensureOpened(conid);
+    if (opened.connection.isReadOnly) return false;
+    return this.loadDataCore('summaryCommand', { conid, command, row });
+  },
 };

packages/api/src/controllers/sessions.js

@@ -4,7 +4,15 @@ const connections = require('./connections');
 const socket = require('../utility/socket');
 const { fork } = require('child_process');
 const jsldata = require('./jsldata');
+const path = require('path');
 const { handleProcessCommunication } = require('../utility/processComm');
+const processArgs = require('../utility/processArgs');
+const { appdir } = require('../utility/directories');
+const { getLogger } = require('dbgate-tools');
+const pipeForkLogs = require('../utility/pipeForkLogs');
+const config = require('./config');
+
+const logger = getLogger('sessions');
 
 module.exports = {
   /** @type {import('dbgate-types').OpenedSession[]} */
@@ -45,9 +53,18 @@ module.exports = {
     this.dispatchMessage(sesid, info);
   },
 
-  handle_done(sesid) {
+  handle_done(sesid, props) {
     socket.emit(`session-done-${sesid}`);
-    this.dispatchMessage(sesid, 'Query execution finished');
+    if (!props.skipFinishedMessage) {
+      this.dispatchMessage(sesid, 'Query execution finished');
+    }
+    const session = this.opened.find(x => x.sesid == sesid);
+    if (session.loadingReader_jslid) {
+      socket.emit(`session-jslid-done-${session.loadingReader_jslid}`);
+    }
+    if (session.killOnDone) {
+      this.kill({ sesid });
+    }
   },
 
   handle_recordset(sesid, props) {
@@ -59,13 +76,31 @@ module.exports = {
     jsldata.notifyChangedStats(stats);
   },
 
+  handle_initializeFile(sesid, props) {
+    const { jslid } = props;
+    socket.emit(`session-initialize-file-${jslid}`);
+  },
+
   handle_ping() {},
 
-  create_meta: 'post',
+  create_meta: true,
   async create({ conid, database }) {
     const sesid = uuidv1();
-    const connection = await connections.get({ conid });
-    const subprocess = fork(process.argv[1], ['sessionProcess', ...process.argv.slice(3)]);
+    const connection = await connections.getCore({ conid });
+    const subprocess = fork(
+      global['API_PACKAGE'] || process.argv[1],
+      [
+        '--is-forked-api',
+        '--start-process',
+        'sessionProcess',
+        ...processArgs.getPassArgs(),
+        // ...process.argv.slice(3),
+      ],
+      {
+        stdio: ['ignore', 'pipe', 'pipe', 'ipc'],
+      }
+    );
+    pipeForkLogs(subprocess);
     const newOpened = {
       conid,
       database,
@@ -80,25 +115,84 @@ module.exports = {
       if (handleProcessCommunication(message, subprocess)) return;
       this[`handle_${msgtype}`](sesid, message);
     });
-    subprocess.send({ msgtype: 'connect', ...connection, database });
-    return newOpened;
+    subprocess.on('exit', () => {
+      this.opened = this.opened.filter(x => x.sesid != sesid);
+      this.dispatchMessage(sesid, 'Query session closed');
+      socket.emit(`session-closed-${sesid}`);
+    });
+
+    subprocess.send({
+      msgtype: 'connect',
+      ...connection,
+      database,
+      globalSettings: await config.getSettings(),
+    });
+    return _.pick(newOpened, ['conid', 'database', 'sesid']);
   },
 
-  executeQuery_meta: 'post',
+  executeQuery_meta: true,
   async executeQuery({ sesid, sql }) {
     const session = this.opened.find(x => x.sesid == sesid);
     if (!session) {
       throw new Error('Invalid session');
     }
 
-    console.log(`Processing query, sesid=${sesid}, sql=${sql}`);
+    logger.info({ sesid, sql }, 'Processing query');
     this.dispatchMessage(sesid, 'Query execution started');
     session.subprocess.send({ msgtype: 'executeQuery', sql });
 
     return { state: 'ok' };
   },
 
-  // cancel_meta: 'post',
+  executeReader_meta: true,
+  async executeReader({ conid, database, sql, queryName, appFolder }) {
+    const { sesid } = await this.create({ conid, database });
+    const session = this.opened.find(x => x.sesid == sesid);
+    session.killOnDone = true;
+    const jslid = uuidv1();
+    session.loadingReader_jslid = jslid;
+    const fileName = queryName && appFolder ? path.join(appdir(), appFolder, `${queryName}.query.sql`) : null;
+
+    session.subprocess.send({ msgtype: 'executeReader', sql, fileName, jslid });
+
+    return { jslid };
+  },
+
+  stopLoadingReader_meta: true,
+  async stopLoadingReader({ jslid }) {
+    const session = this.opened.find(x => x.loadingReader_jslid == jslid);
+    if (session) {
+      this.kill({ sesid: session.sesid });
+    }
+    return true;
+  },
+
+  startProfiler_meta: true,
+  async startProfiler({ sesid }) {
+    const jslid = uuidv1();
+    const session = this.opened.find(x => x.sesid == sesid);
+    if (!session) {
+      throw new Error('Invalid session');
+    }
+
+    logger.info({ sesid }, 'Starting profiler');
+    session.loadingReader_jslid = jslid;
+    session.subprocess.send({ msgtype: 'startProfiler', jslid });
+
+    return { state: 'ok', jslid };
+  },
+
+  stopProfiler_meta: true,
+  async stopProfiler({ sesid }) {
+    const session = this.opened.find(x => x.sesid == sesid);
+    if (!session) {
+      throw new Error('Invalid session');
+    }
+    session.subprocess.send({ msgtype: 'stopProfiler' });
+    return { state: 'ok' };
+  },
+
+  // cancel_meta: true,
   // async cancel({ sesid }) {
   //   const session = this.opened.find((x) => x.sesid == sesid);
   //   if (!session) {
@@ -108,7 +202,7 @@ module.exports = {
   //   return { state: 'ok' };
   // },
 
-  kill_meta: 'post',
+  kill_meta: true,
   async kill({ sesid }) {
     const session = this.opened.find(x => x.sesid == sesid);
     if (!session) {
@@ -119,7 +213,27 @@ module.exports = {
     return { state: 'ok' };
   },
 
-  // runCommand_meta: 'post',
+  ping_meta: true,
+  async ping({ sesid }) {
+    const session = this.opened.find(x => x.sesid == sesid);
+    if (!session) {
+      throw new Error('Invalid session');
+    }
+    try {
+      session.subprocess.send({ msgtype: 'ping' });
+    } catch (err) {
+      logger.error({ err }, 'Error pinging session');
+
+      return {
+        status: 'error',
+        message: 'Ping failed',
+      };
+    }
+
+    return { state: 'ok' };
+  },
+
+  // runCommand_meta: true,
   // async runCommand({ conid, database, sql }) {
   //   console.log(`Running SQL command , conid=${conid}, database=${database}, sql=${sql}`);
   //   const opened = await this.ensureOpened(conid, database);

packages/api/src/controllers/uploads.js

@@ -1,6 +1,8 @@
 const path = require('path');
 const { uploadsdir } = require('../utility/directories');
 const uuidv1 = require('uuid/v1');
+const { getLogger } = require('dbgate-tools');
+const logger = getLogger('uploads');
 
 module.exports = {
   upload_meta: {
@@ -15,7 +17,7 @@ module.exports = {
     }
     const uploadName = uuidv1();
     const filePath = path.join(uploadsdir(), uploadName);
-    console.log(`Uploading file ${data.name}, size=${data.size}`);
+    logger.info(`Uploading file ${data.name}, size=${data.size}`);
 
     data.mv(filePath, () => {
       res.json({
@@ -25,4 +27,12 @@ module.exports = {
       });
     });
   },
+
+  get_meta: {
+    method: 'get',
+    raw: true,
+  },
+  get(req, res) {
+    res.sendFile(path.join(uploadsdir(), req.query.file));
+  },
 };

packages/api/src/currentVersion.js

@@ -1,5 +1,5 @@
 
 module.exports = { 
-  version: '3.9.5',
-  buildTime: '2021-02-08T18:21:44.182Z'
+  version: '5.0.0-alpha.1',
+  buildTime: '2021-04-17T07:22:49.702Z'
 };

packages/api/src/index.js

@@ -1,18 +1,115 @@
+const { setLogger, getLogger, setLoggerName } = require('dbgate-tools');
+const processArgs = require('./utility/processArgs');
+const fs = require('fs');
+const moment = require('moment');
+const path = require('path');
+const { logsdir, setLogsFilePath, getLogsFilePath } = require('./utility/directories');
+const { createLogger } = require('pinomin');
+
+if (processArgs.startProcess) {
+  setLoggerName(processArgs.startProcess.replace(/Process$/, ''));
+}
+if (processArgs.processDisplayName) {
+  setLoggerName(processArgs.processDisplayName);
+}
+
+// function loadLogsContent(maxLines) {
+//   const text = fs.readFileSync(getLogsFilePath(), { encoding: 'utf8' });
+//   if (maxLines) {
+//     const lines = text
+//       .split('\n')
+//       .map(x => x.trim())
+//       .filter(x => x);
+//     return lines.slice(-maxLines).join('\n');
+//   }
+//   return text;
+// }
+
+function configureLogger() {
+  const logsFilePath = path.join(logsdir(), `${moment().format('YYYY-MM-DD-HH-mm')}-${process.pid}.ndjson`);
+  setLogsFilePath(logsFilePath);
+  setLoggerName('main');
+
+  const logger = createLogger({
+    base: { pid: process.pid },
+    targets: [
+      {
+        type: 'console',
+        // @ts-ignore
+        level: process.env.CONSOLE_LOG_LEVEL || process.env.LOG_LEVEL || 'info',
+      },
+      {
+        type: 'stream',
+        // @ts-ignore
+        level: process.env.FILE_LOG_LEVEL || process.env.LOG_LEVEL || 'info',
+        stream: fs.createWriteStream(logsFilePath, { flags: 'a' }),
+      },
+    ],
+  });
+
+  // const streams = [];
+  // if (!platformInfo.isElectron) {
+  //   streams.push({
+  //     stream: process.stdout,
+  //     level: process.env.CONSOLE_LOG_LEVEL || process.env.LOG_LEVEL || 'info',
+  //   });
+  // }
+
+  // streams.push({
+  //   stream: fs.createWriteStream(logsFilePath),
+  //   level: process.env.FILE_LOG_LEVEL || process.env.LOG_LEVEL || 'info',
+  // });
+
+  // let logger = pinoms({
+  //   redact: { paths: ['hostname'], remove: true },
+  //   streams,
+  // });
+
+  // // @ts-ignore
+  // let logger = pino({
+  //   redact: { paths: ['hostname'], remove: true },
+  //   transport: {
+  //     targets: [
+  //       {
+  //         level: process.env.CONSOLE_LOG_LEVEL || process.env.LOG_LEVEL || 'info',
+  //         target: 'pino/file',
+  //       },
+  //       {
+  //         level: process.env.FILE_LOG_LEVEL || process.env.LOG_LEVEL || 'info',
+  //         target: 'pino/file',
+  //         options: { destination: logsFilePath },
+  //       },
+  //     ],
+  //   },
+  // });
+
+  setLogger(logger);
+}
+
+if (processArgs.listenApi) {
+  configureLogger();
+}
+
 const shell = require('./shell');
+const dbgateTools = require('dbgate-tools');
 
-const argument = process.argv[2];
-if (argument && argument.endsWith('Process')) {
+global['DBGATE_TOOLS'] = dbgateTools;
+
+if (processArgs.startProcess) {
   const proc = require('./proc');
-
-  const module = proc[argument];
+  const module = proc[processArgs.startProcess];
   module.start();
-} else if (!module['parent'] && !process.argv.includes('--checkParent')) {
-  const main = require('./main');
+}
 
-  main.start(argument);
+if (processArgs.listenApi) {
+  const main = require('./main');
+  main.start();
 }
 
 module.exports = {
   ...shell,
+  getLogger,
+  configureLogger,
+  // loadLogsContent,
   getMainModule: () => require('./main'),
 };

packages/api/src/main.js

@@ -4,10 +4,7 @@ const bodyParser = require('body-parser');
 const fileUpload = require('express-fileupload');
 const http = require('http');
 const cors = require('cors');
-const io = require('socket.io');
-const fs = require('fs');
-const findFreePort = require('find-free-port');
-const childProcessChecker = require('./utility/childProcessChecker');
+const getPort = require('get-port');
 const path = require('path');
 
 const useController = require('./utility/useController');
@@ -22,27 +19,36 @@ const runners = require('./controllers/runners');
 const jsldata = require('./controllers/jsldata');
 const config = require('./controllers/config');
 const archive = require('./controllers/archive');
+const apps = require('./controllers/apps');
+const auth = require('./controllers/auth');
 const uploads = require('./controllers/uploads');
 const plugins = require('./controllers/plugins');
 const files = require('./controllers/files');
 const scheduler = require('./controllers/scheduler');
+const queryHistory = require('./controllers/queryHistory');
+const onFinished = require('on-finished');
 
 const { rundir } = require('./utility/directories');
+const platformInfo = require('./utility/platformInfo');
+const getExpressPath = require('./utility/getExpressPath');
+const { getLogins } = require('./utility/hasPermission');
+const _ = require('lodash');
+const { getLogger } = require('dbgate-tools');
 
-function start(argument = null) {
+const logger = getLogger('main');
+
+function start() {
   // console.log('process.argv', process.argv);
 
   const app = express();
 
   const server = http.createServer(app);
-  socket.set(io(server));
 
-  if (process.env.LOGIN && process.env.PASSWORD) {
+  const logins = getLogins();
+  if (logins && process.env.BASIC_AUTH) {
     app.use(
       basicAuth({
-        users: {
-          [process.env.LOGIN]: process.env.PASSWORD,
-        },
+        users: _.fromPairs(logins.map(x => [x.login, x.password])),
         challenge: true,
         realm: 'DbGate Web App',
       })
@@ -50,65 +56,123 @@ function start(argument = null) {
   }
 
   app.use(cors());
+
+  if (platformInfo.isDocker) {
+    // server static files inside docker container
+    app.use(getExpressPath('/'), express.static('/home/dbgate-docker/public'));
+  } else if (platformInfo.isNpmDist) {
+    app.use(getExpressPath('/'), express.static(path.join(__dirname, '../../dbgate-web/public')));
+  } else if (process.env.DEVWEB) {
+    // console.log('__dirname', __dirname);
+    // console.log(path.join(__dirname, '../../web/public/build'));
+    app.use(getExpressPath('/'), express.static(path.join(__dirname, '../../web/public')));
+  } else {
+    app.get(getExpressPath('/'), (req, res) => {
+      res.send('DbGate API');
+    });
+  }
+
+  if (auth.shouldAuthorizeApi()) {
+    app.use(auth.authMiddleware);
+  }
+
+  app.get(getExpressPath('/stream'), async function (req, res) {
+    res.set({
+      'Cache-Control': 'no-cache',
+      'Content-Type': 'text/event-stream',
+      'X-Accel-Buffering': 'no',
+      Connection: 'keep-alive',
+    });
+    res.flushHeaders();
+
+    // Tell the client to retry every 10 seconds if connectivity is lost
+    res.write('retry: 10000\n\n');
+    socket.addSseResponse(res);
+    onFinished(req, () => {
+      socket.removeSseResponse(res);
+    });
+  });
+
   app.use(bodyParser.json({ limit: '50mb' }));
 
   app.use(
-    '/uploads',
+    getExpressPath('/uploads'),
     fileUpload({
       limits: { fileSize: 4 * 1024 * 1024 },
     })
   );
 
-  useController(app, '/connections', connections);
-  useController(app, '/server-connections', serverConnections);
-  useController(app, '/database-connections', databaseConnections);
-  useController(app, '/metadata', metadata);
-  useController(app, '/sessions', sessions);
-  useController(app, '/runners', runners);
-  useController(app, '/jsldata', jsldata);
-  useController(app, '/config', config);
-  useController(app, '/archive', archive);
-  useController(app, '/uploads', uploads);
-  useController(app, '/plugins', plugins);
-  useController(app, '/files', files);
-  useController(app, '/scheduler', scheduler);
+  useAllControllers(app, null);
 
   // if (process.env.PAGES_DIRECTORY) {
   //   app.use('/pages', express.static(process.env.PAGES_DIRECTORY));
   // }
 
-  app.use('/runners/data', express.static(rundir()));
+  app.use(getExpressPath('/runners/data'), express.static(rundir()));
 
-  if (fs.existsSync('/home/dbgate-docker/build')) {
-    // server static files inside docker container
-    app.use(express.static('/home/dbgate-docker/build'));
-  } else {
-    if (argument != 'startNodeWeb') {
-      app.get('/', (req, res) => {
-        res.send('DbGate API');
-      });
-    }
-  }
-
-  if (argument == '--dynport') {
-    childProcessChecker();
-
-    findFreePort(53911, function (err, port) {
+  if (platformInfo.isDocker) {
+    const port = process.env.PORT || 3000;
+    logger.info(`DbGate API listening on port ${port} (docker build)`);
+    server.listen(port);
+  } else if (platformInfo.isNpmDist) {
+    getPort({
+      port: parseInt(
+        // @ts-ignore
+        process.env.PORT || 3000
+      ),
+    }).then(port => {
       server.listen(port, () => {
-        console.log(`DbGate API listening on port ${port}`);
-        process.send({ msgtype: 'listening', port });
-      });
-    });
-  } else if (argument == 'startNodeWeb') {
-    app.use(express.static(path.join(__dirname, '../../dbgate-web/build')));
-    findFreePort(5000, function (err, port) {
-      server.listen(port, () => {
-        console.log(`DbGate API listening on port ${port}`);
+        logger.info(`DbGate API listening on port ${port} (NPM build)`);
       });
     });
+  } else if (process.env.DEVWEB) {
+    const port = process.env.PORT || 3000;
+    logger.info(`DbGate API & web listening on port ${port} (dev web build)`);
+    server.listen(port);
   } else {
-    server.listen(3000);
+    const port = process.env.PORT || 3000;
+    logger.info(`DbGate API listening on port ${port} (dev API build)`);
+    server.listen(port);
   }
+
+  function shutdown() {
+    logger.info('\nShutting down DbGate API server');
+    server.close(() => {
+      logger.info('Server shut down, terminating');
+      process.exit(0);
+    });
+    setTimeout(() => {
+      logger.info('Server close timeout, terminating');
+      process.exit(0);
+    }, 1000);
+  }
+
+  process.on('SIGINT', shutdown);
+  process.on('SIGTERM', shutdown);
+  process.on('SIGBREAK', shutdown);
 }
 
-module.exports = { start };
+function useAllControllers(app, electron) {
+  useController(app, electron, '/connections', connections);
+  useController(app, electron, '/server-connections', serverConnections);
+  useController(app, electron, '/database-connections', databaseConnections);
+  useController(app, electron, '/metadata', metadata);
+  useController(app, electron, '/sessions', sessions);
+  useController(app, electron, '/runners', runners);
+  useController(app, electron, '/jsldata', jsldata);
+  useController(app, electron, '/config', config);
+  useController(app, electron, '/archive', archive);
+  useController(app, electron, '/uploads', uploads);
+  useController(app, electron, '/plugins', plugins);
+  useController(app, electron, '/files', files);
+  useController(app, electron, '/scheduler', scheduler);
+  useController(app, electron, '/query-history', queryHistory);
+  useController(app, electron, '/apps', apps);
+  useController(app, electron, '/auth', auth);
+}
+
+function setElectronSender(electronSender) {
+  socket.setElectronSender(electronSender);
+}
+
+module.exports = { start, useAllControllers, setElectronSender, configController: config };

packages/api/src/nativeModules.js

@@ -1,5 +1,13 @@
 const argIndex = process.argv.indexOf('--native-modules');
-const redirectFile = argIndex > 0 ? process.argv[argIndex + 1] : null;
+const redirectFile = global['NATIVE_MODULES'] || (argIndex > 0 ? process.argv[argIndex + 1] : null);
 
-// @ts-ignore
-module.exports = redirectFile ? __non_webpack_require__(redirectFile) : require('./nativeModulesContent');
+function requireDynamic(file) {
+  try {
+    // @ts-ignore
+    return __non_webpack_require__(redirectFile);
+  } catch (err) {
+    return require(redirectFile);
+  }
+}
+
+module.exports = redirectFile ? requireDynamic(redirectFile) : require('./nativeModulesContent');