v5.2.5-beta.16

fix: connection ssl require file path instread of file content

.github/ISSUE_TEMPLATE/bug_report.md

@@ -28,9 +28,3 @@ If applicable, add screenshots to help explain your problem.
  - Install source [e.g. installer/SNAP/Docker/NPM]
  - Type - Web/Application
  - Database engine: [e.g. MySQL/PostgreSQL/SQL Server]
-
-**Additional info**
-You could provide additional context, to better understand your case.
- - How looks your typical DbGate usage?
- - What other DB software do you use?
- - Anything else you think might be helpful

.github/ISSUE_TEMPLATE/feature_request.md

@@ -15,9 +15,3 @@ A clear and concise description of what you want to happen.
 
 **Describe alternatives you've considered**
 A clear and concise description of any alternative solutions or features you've considered.
-
-**Additional info**
-You could provide additional context, to better understand your case.
- - How looks your typical DbGate usage?
- - What other DB software do you use?
- - Anything else you think might be helpful

.github/ISSUE_TEMPLATE/question.md

@@ -16,9 +16,3 @@ App Version [help -> about]:
 
 **Screenshot [if appropriate]**:
 A screenshot of the app if that helps
-
-**Additional info**
-You could provide additional context, to better understand your case.
- - How looks your typical DbGate usage?
- - What other DB software do you use?
- - Anything else you think might be helpful

.github/workflows/build-app-beta.yaml

@@ -10,9 +10,10 @@ jobs:
     runs-on: ${{ matrix.os }}
 
     strategy:
+      fail-fast: false
       matrix:
-        # os: [ubuntu-18.04, windows-2016]
-        os: [macOS-10.15, windows-2016, ubuntu-18.04]
+        os: [macOS-10.15, windows-2022, ubuntu-22.04]
+        # os: [macOS-10.15]
 
     steps:
       - name: Context
@@ -26,6 +27,12 @@ jobs:
         uses: actions/setup-node@v1
         with:
           node-version: 14.x
+      - name: yarn adjustPackageJson
+        run: |
+          yarn adjustPackageJson
+      - name: yarn set timeout
+        run: |
+          yarn config set network-timeout 100000
       - name: yarn install
         run: |
           yarn install
@@ -39,28 +46,31 @@ jobs:
         run: |
           yarn fillPackagedPlugins
       - name: Install Snapcraft
-        if: matrix.os == 'ubuntu-18.04'
+        if: matrix.os == 'ubuntu-22.04'
         uses: samuelmeuli/action-snapcraft@v1
       - name: Publish
         run: |
           yarn run build:app
         env:
           GH_TOKEN: ${{ secrets.GH_TOKEN }} # token for electron publish
-          WIN_CSC_LINK: ${{ secrets.WINCERT_CERTIFICATE }}
-          WIN_CSC_KEY_PASSWORD: ${{ secrets.WINCERT_PASSWORD }}
 
-      - name: Save snap login
-        if: matrix.os == 'ubuntu-18.04'
-        run: 'echo "$SNAPCRAFT_LOGIN" > snapcraft.login'
-        shell: bash
-        env:
-          SNAPCRAFT_LOGIN: ${{secrets.SNAPCRAFT_LOGIN}}
+          WIN_CSC_LINK: ${{ secrets.WINCERT_2025 }}
+          WIN_CSC_KEY_PASSWORD: ${{ secrets.WINCERT_2025_PASSWORD }}
+          # WIN_CSC_LINK: ${{ secrets.WINCERT_CERTIFICATE }}
+          # WIN_CSC_KEY_PASSWORD: ${{ secrets.WINCERT_PASSWORD }}
+
+          CSC_LINK: ${{ secrets.APPLECERT_CERTIFICATE }}
+          CSC_KEY_PASSWORD: ${{ secrets.APPLECERT_PASSWORD }}
+
+          APPLE_ID: ${{ secrets.APPLE_ID }}
+          APPLE_ID_PASSWORD: ${{ secrets.APPLE_ID_PASSWORD }}
 
       - name: publishSnap
-        if: matrix.os == 'ubuntu-18.04'
+        if: matrix.os == 'ubuntu-22.04'
         run: |
-          snapcraft login --with snapcraft.login
           snapcraft upload --release=beta app/dist/*.snap
+        env:
+          SNAPCRAFT_STORE_CREDENTIALS: ${{secrets.SNAPCRAFT_LOGIN}}
 
       - name: Copy artifacts
         run: |

.github/workflows/build-app.yaml

@@ -14,9 +14,10 @@ jobs:
     runs-on: ${{ matrix.os }}
 
     strategy:
+      fail-fast: false
       matrix:
-        # os: [ubuntu-18.04, windows-2016]
-        os: [macOS-10.15, windows-2016, ubuntu-18.04]
+        # os: [ubuntu-22.04, windows-2016]
+        os: [macOS-10.15, windows-2022, ubuntu-22.04]
 
     steps:
       - name: Context
@@ -30,6 +31,12 @@ jobs:
         uses: actions/setup-node@v1
         with:
           node-version: 14.x
+      - name: yarn adjustPackageJson
+        run: |
+          yarn adjustPackageJson
+      - name: yarn set timeout
+        run: |
+          yarn config set network-timeout 100000
       - name: yarn install
         run: |
           # yarn --version
@@ -45,32 +52,35 @@ jobs:
         run: |
           yarn fillPackagedPlugins
       - name: Install Snapcraft
-        if: matrix.os == 'ubuntu-18.04'
+        if: matrix.os == 'ubuntu-22.04'
         uses: samuelmeuli/action-snapcraft@v1
       - name: Publish
         run: |
           yarn run build:app
         env:
           GH_TOKEN: ${{ secrets.GH_TOKEN }} # token for electron publish
-          WIN_CSC_LINK: ${{ secrets.WINCERT_CERTIFICATE }}
-          WIN_CSC_KEY_PASSWORD: ${{ secrets.WINCERT_PASSWORD }}
+
+          WIN_CSC_LINK: ${{ secrets.WINCERT_2025 }}
+          WIN_CSC_KEY_PASSWORD: ${{ secrets.WINCERT_2025_PASSWORD }}
+          # WIN_CSC_LINK: ${{ secrets.WINCERT_CERTIFICATE }}
+          # WIN_CSC_KEY_PASSWORD: ${{ secrets.WINCERT_PASSWORD }}
+
+          CSC_LINK: ${{ secrets.APPLECERT_CERTIFICATE }}
+          CSC_KEY_PASSWORD: ${{ secrets.APPLECERT_PASSWORD }}
+
+          APPLE_ID: ${{ secrets.APPLE_ID }}
+          APPLE_ID_PASSWORD: ${{ secrets.APPLE_ID_PASSWORD }}
 
       - name: generatePadFile
         run: |
           yarn generatePadFile
 
-      - name: Save snap login
-        if: matrix.os == 'ubuntu-18.04'
-        run: 'echo "$SNAPCRAFT_LOGIN" > snapcraft.login'
-        shell: bash
-        env:
-          SNAPCRAFT_LOGIN: ${{secrets.SNAPCRAFT_LOGIN}}
-
       - name: publishSnap
-        if: matrix.os == 'ubuntu-18.04'
+        if: matrix.os == 'ubuntu-22.04'
         run: |
-          snapcraft login --with snapcraft.login
           snapcraft upload --release=stable app/dist/*.snap
+        env:
+          SNAPCRAFT_STORE_CREDENTIALS: ${{secrets.SNAPCRAFT_LOGIN}}
 
       - name: Copy artifacts
         run: |
@@ -83,8 +93,8 @@ jobs:
           cp app/dist/*.exe artifacts/dbgate-latest.exe || true
           cp app/dist/*win_x64.zip artifacts/dbgate-windows-latest.zip || true
           cp app/dist/*win_arm64.zip artifacts/dbgate-windows-latest-arm64.zip || true
-          cp app/dist/*-mac_x64.dmg artifacts/dbgate-latest.dmg || true
-          cp app/dist/*-mac_arm64.dmg artifacts/dbgate-latest-arm64.dmg || true
+          cp app/dist/*-mac_universal.dmg artifacts/dbgate-latest.dmg || true
+          cp app/dist/*-mac_x64.dmg artifacts/dbgate-latest-x64.dmg || true
 
           mv app/dist/*.exe artifacts/ || true
           mv app/dist/*.zip artifacts/ || true
@@ -118,13 +128,13 @@ jobs:
       #     mv app/dist/latest.yml artifacts/latest.yml || true
 
       - name: Copy latest.yml (windows)
-        if: matrix.os == 'windows-2016'
+        if: matrix.os == 'windows-2022'
         run: |
           mv app/dist/latest.yml artifacts/latest.yml || true
           mv app/dist/dbgate-pad.xml artifacts/ || true
 
       - name: Copy latest-linux.yml
-        if: matrix.os == 'ubuntu-18.04'
+        if: matrix.os == 'ubuntu-22.04'
         run: |
           mv app/dist/latest-linux.yml artifacts/latest-linux.yml || true
 

.github/workflows/build-docker-beta.yaml

@@ -1,58 +0,0 @@
-name: Docker image BETA
-
-# on: [push]
-
-on:
-  push:
-    tags:
-      - 'v[0-9]+.[0-9]+.[0-9]+-beta.[0-9]+'
-      - 'v[0-9]+.[0-9]+.[0-9]+-docker.[0-9]+'
-
-jobs:
-  build:
-
-    runs-on: ${{ matrix.os }}
-
-    strategy:
-      matrix:
-        os: [ubuntu-18.04]
-
-    steps:
-      - name: Context
-        env:
-          GITHUB_CONTEXT: ${{ toJson(github) }}
-        run: echo "$GITHUB_CONTEXT"
-      - uses: actions/checkout@v2
-        with:
-          fetch-depth: 1
-      - name: Use Node.js 14.x
-        uses: actions/setup-node@v1
-        with:
-          node-version: 14.x
-      - name: yarn install
-        run: |
-          # yarn --version
-          # yarn config set network-timeout 300000
-          yarn install
-      - name: setCurrentVersion
-        run: |
-          yarn setCurrentVersion
-      - name: Prepare docker image
-        run: |
-          yarn run prepare:docker
-      - name: Build docker image
-        run: |
-          docker build ./docker -t dbgate
-      - name: Push docker image
-        run: |
-          docker tag dbgate dbgate/dbgate:beta
-          docker login -u ${{ secrets.DOCKER_USERNAME }} -p ${{ secrets.DOCKER_PASSWORD }}
-          docker push dbgate/dbgate:beta
-      - name: Build alpine docker image
-        run: |
-          docker build ./docker -t dbgate -f docker/Dockerfile-alpine
-      - name: Push alpine docker image
-        run: |
-          docker tag dbgate dbgate/dbgate:beta-alpine
-          docker login -u ${{ secrets.DOCKER_USERNAME }} -p ${{ secrets.DOCKER_PASSWORD }}
-          docker push dbgate/dbgate:beta-alpine

.github/workflows/build-docker.yaml

@@ -1,17 +1,11 @@
 name: Docker image
 
-# on: [push]
-
 on:
   push:
     tags:
       - 'v[0-9]+.[0-9]+.[0-9]+'
-    # - 'v*' # Push events to matching v*, i.e. v1.0, v20.15.10
-
-# on:
-#   push:
-#     branches:
-#       - production
+      - 'v[0-9]+.[0-9]+.[0-9]+-beta.[0-9]+'
+      - 'v[0-9]+.[0-9]+.[0-9]+-docker.[0-9]+'
 
 jobs:
   build:
@@ -20,7 +14,7 @@ jobs:
 
     strategy:
       matrix:
-        os: [ubuntu-18.04]
+        os: [ubuntu-22.04]
 
     steps:
       - name: Context
@@ -30,12 +24,43 @@ jobs:
       - uses: actions/checkout@v2
         with:
           fetch-depth: 1
+
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v4
+        with:
+          images: |
+            dbgate/dbgate
+          flavor: |
+            latest=false
+          tags: |
+            type=raw,value=beta,enable=${{ contains(github.ref_name, '-docker.') || contains(github.ref_name, '-beta.') }}
+
+            type=match,pattern=\d+.\d+.\d+,enable=${{ !contains(github.ref_name, '-docker.') && !contains(github.ref_name, '-beta.') }}
+            type=raw,value=latest,enable=${{ !contains(github.ref_name, '-docker.') && !contains(github.ref_name, '-beta.') }}
+
+      - name: Docker alpine meta
+        id: alpmeta
+        uses: docker/metadata-action@v4
+        with:
+          images: |
+            dbgate/dbgate
+          flavor: |
+            latest=false
+          tags: |
+            type=raw,value=beta-alpine,enable=${{ contains(github.ref_name, '-docker.') || contains(github.ref_name, '-beta.') }}
+
+            type=match,pattern=\d+.\d+.\d+,suffix=-alpine,enable=${{ !contains(github.ref_name, '-docker.') && !contains(github.ref_name, '-beta.') }}
+            type=raw,value=alpine,enable=${{ !contains(github.ref_name, '-docker.') && !contains(github.ref_name, '-beta.') }}
+
       - name: Use Node.js 14.x
         uses: actions/setup-node@v1
         with:
           node-version: 14.x
       - name: yarn install
         run: |
+          # yarn --version
+          # yarn config set network-timeout 300000
           yarn install
       - name: setCurrentVersion
         run: |
@@ -43,19 +68,28 @@ jobs:
       - name: Prepare docker image
         run: |
           yarn run prepare:docker
-      - name: Build docker image
-        run: |
-          docker build ./docker -t dbgate
-      - name: Push docker image
-        run: |
-          docker tag dbgate dbgate/dbgate
-          docker login -u ${{ secrets.DOCKER_USERNAME }} -p ${{ secrets.DOCKER_PASSWORD }}
-          docker push dbgate/dbgate
-      - name: Build alpine docker image
-        run: |
-          docker build ./docker -t dbgate -f docker/Dockerfile-alpine
-      - name: Push alpine docker image
-        run: |
-          docker tag dbgate dbgate/dbgate:alpine
-          docker login -u ${{ secrets.DOCKER_USERNAME }} -p ${{ secrets.DOCKER_PASSWORD }}
-          docker push dbgate/dbgate:alpine
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+      - name: Login to DockerHub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Build and push
+        uses: docker/build-push-action@v3
+        with:
+          push: true
+          context: ./docker
+          tags: ${{ steps.meta.outputs.tags }}
+          platforms: linux/amd64,linux/arm64,linux/arm/v7
+
+      - name: Build and push alpine
+        uses: docker/build-push-action@v3
+        with:
+          push: true
+          context: ./docker
+          file: ./docker/Dockerfile-alpine
+          tags: ${{ steps.alpmeta.outputs.tags }}
+          platforms: linux/amd64,linux/arm64,linux/arm/v7

.github/workflows/build-npm.yaml

@@ -20,7 +20,7 @@ jobs:
 
     strategy:
       matrix:
-        os: [ubuntu-18.04]
+        os: [ubuntu-22.04]
 
     steps:
       - name: Context
@@ -79,21 +79,26 @@ jobs:
         run: |
           npm publish
 
-      - name: Publish query-splitter
-        working-directory: packages/query-splitter
-        run: |
-          npm publish
-          
       - name: Publish web
         working-directory: packages/web
         run: |
           npm publish
-                                        
-      - name: Publish dbgate
+
+      - name: Publish dbgate (obsolete)
         working-directory: packages/dbgate
         run: |
           npm publish
 
+      - name: Publish dbgate-serve
+        working-directory: packages/serve
+        run: |
+          npm publish
+
+      - name: Publish dbmodel
+        working-directory: packages/dbmodel
+        run: |
+          npm publish
+
       - name: Publish dbgate-plugin-csv
         working-directory: plugins/dbgate-plugin-csv
         run: |
@@ -133,3 +138,13 @@ jobs:
         working-directory: plugins/dbgate-plugin-sqlite
         run: |
           npm publish
+
+      - name: Publish dbgate-plugin-redis
+        working-directory: plugins/dbgate-plugin-redis
+        run: |
+          npm publish
+
+      - name: Publish dbgate-plugin-oracle
+        working-directory: plugins/dbgate-plugin-oracle
+        run: |
+          npm publish

.github/workflows/run-tests.yaml

@@ -31,10 +31,10 @@ jobs:
         run: |
           cd packages/filterparser
           yarn test:ci
-      - name: Query spliiter tests
+      - name: Datalib (perspective) tests
         if: always()
         run: |
-          cd packages/query-splitter
+          cd packages/datalib
           yarn test:ci
       - uses: tanmen/jest-reporter@v1
         if: always()
@@ -52,8 +52,8 @@ jobs:
         if: always()
         with:
           github-token: ${{ secrets.GITHUB_TOKEN }}
-          result-file: packages/query-splitter/result.json
-          action-name: Query splitter test results
+          result-file: packages/datalib/result.json
+          action-name: Datalib (perspectives) test results
 
     services:
       postgres:

.node-version

@@ -0,0 +1 @@
+16.14.2

.vscode/restore-terminals.json

@@ -0,0 +1,20 @@
+{
+  "terminals": [
+    {
+      "splitTerminals": [
+        {
+          "name": "lib",
+          "commands": ["yarn lib"]
+        },
+        {
+          "name": "web",
+          "commands": ["yarn start:web"]
+        },
+        {
+          "name": "api",
+          "commands": ["yarn start:api"]
+        }
+      ]
+    }
+  ]
+}

.vscode/settings.json

@@ -1,3 +1,6 @@
 {
-    "jestrunner.jestCommand": "node_modules/.bin/cross-env DEVMODE=1 LOCALTEST=1 node_modules/.bin/jest"
+    "jestrunner.jestCommand": "node_modules/.bin/cross-env DEVMODE=1 LOCALTEST=1 node_modules/.bin/jest",
+    "cSpell.words": [
+        "dbgate"
+    ]
 }

CHANGELOG.md

@@ -1,5 +1,336 @@
 # ChangeLog
 
+Builds:
+ - docker - build
+ - npm - npm package dbgate-serve
+ - app - classic electron app
+ - mac - application for macOS
+ - linux - application for linux
+ - win - application for Windows
+
+### 5.2.3
+- FIXED: npm version crash (#508)
+
+### 5.2.3
+- ADDED: Search entire table (multi column filter) #491
+- ADDED: OracleDB - connection to toher than default ports #496
+- CHANGED: OracleDB - status of support set to experimental
+- FIXED: OracleDB database URL - fixes: Connect to default Oracle database #489
+- ADDED: HTML, XML code highlighting for Edit cell value #485
+- FIXED: Intellisense - incorrect alias after ORDER BY clause #484
+- FIXED: Typo in SQL-Generator #481
+- ADDED: Data duplicator #480
+- FIXED: MongoDB - support for views #476
+- FIXED: "SQL:CREATE TABLE" generated SQL default value syntax errors #455
+- FIXED: Crash when right-clicking on tables #452
+- FIXED: View sort #436
+- ADDED: Arm64 version for Windows #473
+- ADDED: Sortable query results and data archive
+- CHANGED: Use transactions for saving table data
+- CHANGED: Save table structure uses transactions
+- ADDED: Table data editing - shows editing mark
+- ADDED: Editing data archive files
+- FIXED: Delete cascade options when using more than 2 tables
+- ADDED: Save to current archive commands
+- ADDED: Current archive mark is on status bar
+- FIXED: Changed package used for parsing JSONL files when browsing - fixes backend freezing
+- FIXED: SSL option for mongodb #504
+- REMOVED: Data sheet editor
+- FIXED: Creating SQLite autoincrement column
+- FIXED: Better error reporting from exports/import/dulicator
+- CHANGED: Optimalizede OracleDB analysing algorithm
+- ADDED: Mutli column filter for perspectives
+- FIXED: Fixed some scenarios using tables from different DBs
+- FIXED: Sessions with long-running queries are not killed
+
+
+### 5.2.2
+- FIXED: Optimalized load DB structure for PostgreSQL #451
+- ADDED: Auto-closing query connections after configurable (15 minutes default) no-activity interval #468
+- ADDED: Set application-name connection parameter (for PostgreSQL and MS SQL) for easier identifying of DbGate connections
+- ADDED: Filters supports binary IDs #467
+- FIXED: Ctrl+Tab works (switching tabs) #457
+- FIXED: Format code supports non-standard letters #450
+- ADDED: New logging system, log to file, ability to reduce logging #360 (using https://www.npmjs.com/package/pinomin)
+- FIXED: crash on Windows and Mac after system goes in suspend mode #458
+- ADDED: dbmodel standalone NPM package (https://www.npmjs.com/package/dbmodel) - deploy database via commandline tool
+
+
+### 5.2.1
+- FIXED: client_id param in OAuth
+- ADDED: OAuth scope parameter
+- FIXED: login page - password was not sent, when submitting by pressing ENTER
+- FIXED: Used permissions fix
+- FIXED: Export modal - fixed crash when selecting different database
+
+### 5.2.0
+- ADDED: Oracle database support #380
+- ADDED: OAuth authentification #407
+- ADDED: Active directory (Windows) authentification #261
+- ADDED: Ask database credentials when login to DB
+- ADDED: Login form instead of simple authorization (simple auth is possible with special configuration)
+- FIXED: MongoDB - connection uri regression
+- ADDED: MongoDB server summary tab
+- FIXED: Broken versioned tables in MariaDB #433
+- CHANGED: Improved editor margin #422
+- ADDED: Implemented camel case search in all search boxes
+- ADDED: MonhoDB filter empty array, not empty array
+- ADDED: Maximize button reflects window state
+- ADDED: MongoDB - database profiler
+- CHANGED: Short JSON values are shown directly in grid
+- FIXED: Fixed filtering nested fields in NDJSON viewer
+- CHANGED: Improved fuzzy search after Ctrl+P #246
+- ADDED: MongoDB: Create collection backup
+- ADDED: Single database mode
+- ADDED: Perspective designer supports joins from MongoDB nested documents and arrays
+- FIXED: Perspective designer joins on MongoDB ObjectId fields
+- ADDED: Filtering columns in designer (query designer, diagram designer, perspective designer)
+- FIXED: Clone MongoDB rows without _id attribute #404
+- CHANGED: Improved cell view with GPS latitude, longitude fields
+- ADDED: SQL: ALTER VIEW and SQL:ALTER PROCEDURE scripts
+- ADDED: Ctrl+F5 refreshes data grid also with database structure #428
+- ADDED: Perspective display modes: text, force text #439
+- FIXED: Fixed file filters #445
+- ADDED: Rename, remove connection folder, memoize opened state after app restart #425
+- FIXED: Show SQLServer alter store procedure #435
+
+
+### 5.1.6
+- ADDED: Connection folders support #274
+- ADDED: Keyboard shortcut to hide result window and show/hide the side toolbar #406
+- ADDED: Ability to show/hide query results #406
+- FIXED: Double click does not maximize window on MacOS #416
+- FIXED: Some perspective rendering errors
+- FIXED: Connection to MongoDB via database URL info SSH tunnel is used
+- CHANGED: Updated windows code signing certificate
+- ADDED: Query session cleanup (kill query sessions, if browser tab is closed)
+- CHANGED: More strict timeouts to kill database and server connections (reduces resource consumption)
+
+### 5.1.5
+- ADDED: Support perspectives for MongoDB - MongoDB query designer
+- ADDED: Show JSON content directly in the overview #395
+- CHANGED: OSX Command H shortcut for hiding window #390
+- ADDED: Uppercase Autocomplete Suggestions #389
+- FIXED: Record view left/right arrows cause start record number to be treated as string #388
+- FIXED: MongoDb ObjectId behaviour not consistent in nested objects #387
+- FIXED: demo.dbgate.org - beta version crash 5.1.5-beta.3 #386
+- ADDED: connect via socket - configurable via environment variables #358
+
+### 5.1.4
+- ADDED: Drop database commands #384
+- ADDED: Customizable Redis key separator #379
+- ADDED: ARM support for docker images
+- ADDED: Version tags for docker images
+- ADDED: Better SQL command splitting and highlighting
+- ADDED: Unsaved marker for SQL files
+
+### 5.1.3
+- ADDED: Editing multiline cell values #378 #371 #359
+- ADDED: Truncate table #333
+- ADDED: Perspectives - show row count
+- ADDED: Query - error markers in gutter area
+- ADDED: Query - ability to execute query elements from gutter
+- FIXED: Correct error line numbers returned from queries
+
+### 5.1.2
+- FIXED: MongoDb any export function does not work. #373
+- ADDED: Query Designer short order more flexibility #372
+- ADDED: Form View move between records #370
+- ADDED: Custom SQL conditions in query designer and table filtering #369
+- ADDED: Query Designer filter eq to X or IS NULL #368
+- FIXED: Query designer, open a saved query lost sort order #363
+- ADDED: Query designer reorder columns #362
+- ADDED: connect via socket #358
+- FIXED: Show affected rows after UPDATE/DELETE/INSERT #361
+- ADDED: Perspective cell formatters - JSON, image
+- ADDED: Perspectives - cells without joined data are gray
+
+### 5.1.1
+- ADDED: Perspective designer
+- FIXED: NULL,NOT NULL filter datatime columns #356
+- FIXED: Recognize computed columns on SQL server #354
+- ADDED: Hotkey for clear filter #352
+- FIXED: Change column type on Postgres #350
+- ADDED: Ability to open qdesign file #349
+- ADDED: Custom editor font size #345
+- ADDED: Ability to open perspective files
+
+
+### 5.1.0
+- ADDED: Perspectives (docs: https://dbgate.org/docs/perspectives.html )
+- CHANGED: Upgraded SQLite engine version (driver better-sqlite3: 7.6.2)
+- CHANGED: Upgraded ElectronJS version (from version 13 to version 17)
+- CHANGED: Upgraded all dependencies with current available minor version updates
+- CHANGED: By default, connect on click #332˝
+- CHANGED: Improved keyboard navigation, when editing table data #331
+- ADDED:  Option to skip Save changes dialog #329
+- FIXED: Unsigned column doesn't work correctly. #324
+- FIXED: Connect to MS SQL with domain user now works also under Linux and Mac #305
+
+### 5.0.9
+- FIXED: Fixed problem with SSE events on web version
+- ADDED: Added menu command "New query designer"
+- ADDED: Added menu command "New ER diagram"
+
+### 5.0.8
+- ADDED: SQL Server - support using domain logins under Linux and Mac #305
+- ADDED: Permissions for connections #318
+- ADDED: Ability to change editor front #308
+- ADDED: Custom expression in query designer #306
+- ADDED: OR conditions in query designer #321
+- ADDED: Ability to configure settings view environment variables #304
+ 
+### 5.0.7
+- FIXED: Fixed some problems with SSH tunnel (upgraded SSH client) #315
+- FIXED: Fixed MognoDB executing find query #312
+- ADDED: Interval filters for date/time columns #311
+- ADDED: Ability to clone rows #309
+- ADDED: connecting option Trust server certificate for SQL Server #305
+- ADDED: Autorefresh, reload table every x second #303
+- FIXED(app): Changing editor theme and font size in Editor Themes #300
+ 
+### 5.0.6
+- ADDED: Search in columns
+- CHANGED: Upgraded mongodb driver
+- ADDED: Ability to reset view, when data load fails
+- FIXED: Filtering works for complex types (geography, xml under MSSQL)
+- FIXED: Fixed some NPM package problems
+
+### 5.0.5
+- ADDED: Visualisation geographics objects on map #288
+- ADDED: Support for native SQL as default value inside yaml files #296
+- FIXED: Postgres boolean columns don't filter correctly #298
+- FIXED: Importing dbgate-api as NPM package now works correctly
+- FIXED: Handle error when reading deleted archive
+
+### 5.0.3
+- CHANGED: Optimalization of loading DB structure for PostgreSQL, MySQL #273
+- CHANGED: Upgraded mysql driver #293
+- CHANGED: Better UX when defining SSH port #291
+- ADDED: Database object menu from tab 
+- CHANGED: Ability to close file uploader
+- FIXED: Correct handling of NUL values in update keys
+- CHANGED: Upgraded MS SQL tedious driver
+- ADDED: Change order of pinned tables & databases #227
+- FIXED: #294 Statusbar doesn't match active tab
+- CHANGED: Improved connection worklflow, disconnecting shws confirmations, when it leads to close any tabs
+- ADDED: Configurable object actions #255
+- ADDED: Multiple sort criteria #235
+- ADDED(app): Open JSON file
+### 5.0.2
+- FIXED: Cannot use SSH Tunnel after update #291
+
+### 5.0.1
+- FIXED(app): Can't Click Sidebar Menu Item #287
+
+### 5.0.0
+- CHANGED: Connection workflow, connections are opened on tabs instead of modals
+- ADDED: Posibility to connect to DB without saving connection
+- ADDED(mac): Support for SQLite on Mac M1
+- FIXED(mac): Unable to drag window on MacOS #281 #283
+- CHANGED: Renamed dbgate-data directory to .dbgate #248
+- FIXED: Exported SQL has table name undefined #277
+- ADDED: More data types in table create dialogt #285
+- ADDED(app): Open previously saved ERD diagrams #278
+- CHANGED: Better app loading progress UX
+- FIXED: Removed SSL tab on Redis connection (SSL is not supported for Redis)
+
+### 4.8.8
+- CHANGED: New app icon
+- ADDED: SQL dump, SQL import - also from/to saved queries
+- FIXED(mac): Fixed crash when reopening main window
+- FIXED: MySQL dump now handles correctly dependand views
+- FIXED(app): Browse tabs with Ctrl+Tab
+- ADDED(app): Browse tabs in reverse order with Ctrl+Shift+Tab #245
+
+### 4.8.7
+- ADDED: MySQL dump/backup database
+- ADDED: Import SQL dump from file or from URL
+- FIXED(mac): Fixed Cmd+C, Cmd+V, Cmd+X - shortcuts for copy/cut/paste #270
+- FIXED(mac): Some minor issues on macOS
+- FIXED: Analysing MS SQL nvarchar(max)
+- ADDED: Support for dockerhost network name under docker #271
+
+### 4.8.4
+- FIXED(mac): Fixed build for macOS arm64 #259
+- FIXED(mac): Fixed opening SQLite files on macOS #243
+- FIXED(mac): Fixed opening PEM certificates on macOS #206
+- FIXED(mac): Fixed handling Command key on macOS
+- FIXED(mac): Fixed system menu on macOS
+- FIXED(mac): Fixed reopening main window on macOS
+- CHANGED: Shortcut for net query is now Ctrl+T or Command+T on macOS, former it was Ctrl+Q
+- FIXED: Fixed misplaced tab close icon #260
+- ADDED: Added menu command "Tools/Change to recent database"
+
+### 4.8.3
+- FIXED: filters in query result and NDJSON/archive viewer
+- ADDED: Added select values from query result and NDJSON/archive viewer
+- ADDED: tab navigation in datagrid #254
+- ADDED: Keyboard shortcuts added to help menu #254
+- ADDED: API logging (run enableApiLog() in developers console to enable logging)
+- ADDED: SSH reconnect + moved SSH forward into separate fork #253
+- ADDED: Data type + reference link in column manager
+- FIXED(win,linux,mac): Unable to change theme after installing plugin #244
+
+ ### 4.8.2
+ - ADDED: implemented missing redis search key logic
+
+ ### 4.8.1
+ - FIXED: fixed crash after disconnecting from all DBs
+
+### 4.8.0
+- ADDED: Redis support (support stream type), removed experimental status
+- ADDED: Redis readonly support
+- ADDED: Explicit NDJSON support, when opening NDJSON/JSON lines file, table data are immediately shown, without neccesarity to import
+- ADDED(win,linux,mac): Opening developer tools when crashing without reload app
+### 4.7.4
+- ADDED: Experimental Redis support (full support is planned to version  4.8.0)
+- ADDED: Read-only connections
+- FIXED: MongoDB filters
+- ADDED: MongoDB column value selection
+- ADDED: App related queries
+- ADDED: Fuzzy search #246
+- ADDED(docker, npm): New permissions
+- FIXED(npm): NPM build no longer allocates additonal ports
+- CHANGED(npm): renamed NPM package dbgate => dbgate-serve 
+- CHANGED(docker): custom JavaScripts and connections defined in scripts are now prohibited by default, use SHELL_CONNECTION and SHELL_SCRIPTING environment variables for allowing this
+- ADDED(docker, npm): Better documentation of environment variables configuration, https://dbgate.org/docs/env-variables.html
+- ADDED(docker): support for multiple users with different permissions
+- ADDED(docker): logout operation
+
+### 4.7.3
+- CHANGED: Export menu redesign, quick export menu merged with old export menu
+- REMOVED: Quick export menu
+- ADDED: Export column mapping
+- ADDED: Export invoked from data grid respects columns choosed in column manager
+- ADDED: Quick export (now merged in export menu) is now possible also in web app
+- FIXED: Virtual foreign key editor fixes
+- FIXED: Tabs panel style fix
+- ADDED: Find by schema in databases widget
+- FIXED: Column manager selection fix
+- FIXED: NPM dist - fixed error when loading plugins
+- CHANGED: NPN dist is now executed by dbgate-serve command
+- ADDED: NPM dist accepts .env configuration
+
+### 4.7.2
+- CHANGED: documentation URL - https://dbgate.org/docs/
+- CHANGED: Close button available for all tab groups - #238
+- ADDED: Search function for the Keyboard Shortcuts overview - #239
+- ADDED: Editor font size settings - #229
+- ADDED: Rename MongoDB collection - #223
+- FIXED: bug in cache subsystem
+
+### 4.7.1
+- FIXED: Fixed connecting to MS SQL server running in docker container from DbGate running in docker container #236 
+- FIXED: Fixed export MongoDB collections into Excel and CSV #240
+- ADDED: Added support for docker volumes to persiste connections, when not using configuration via env variables #232
+- ADDED: DbGate in Docker can run in subdirectory #228
+- FIXED: DbGate in Docker can be proxied with nginx #228
+- FIDED: Theme persists when opening multiple windows #207
+- ADDED: Remember fullscreen state #230
+- ADDED: Improved fullscreen state, title bar with menu is hidden, menu is in hamburger menu, like in web version
+- ADDED: Theme choose dialog (added as tab in settings)
+- FIXED: Fixed crash when clicking on application layers #231
 ### 4.7.0
 - CHANGED: Changed main menu style, menu and title bar is in one line (+ability to switch to system menu)
 - REMOVED: Removed main toolbar, use main menu or tab related bottom tool instead

README.md

@@ -1,4 +1,4 @@
-[![NPM version](https://img.shields.io/npm/v/dbgate.svg)](https://www.npmjs.com/package/dbgate)
+[![NPM version](https://img.shields.io/npm/v/dbgate-serve.svg)](https://www.npmjs.com/package/dbgate-serve)
 ![GitHub All Releases](https://img.shields.io/github/downloads/dbgate/dbgate/total) 
 [![dbgate](https://snapcraft.io/dbgate/badge.svg)](https://snapcraft.io/dbgate)
 [![dbgate](https://snapcraft.io/dbgate/trending.svg?name=0)](https://snapcraft.io/dbgate)
@@ -16,13 +16,15 @@ DbGate is licensed under MIT license and is completely free.
 
 * Try it online - [demo.dbgate.org](https://demo.dbgate.org) - online demo application
 * **Download** application for Windows, Linux or Mac from [dbgate.org](https://dbgate.org/download/)
-* Run web version as [NPM package](https://www.npmjs.com/package/dbgate) or as [docker image](https://hub.docker.com/r/dbgate/dbgate)
+* Run web version as [NPM package](https://www.npmjs.com/package/dbgate-serve) or as [docker image](https://hub.docker.com/r/dbgate/dbgate)
 
 ## Supported databases
 * MySQL
 * PostgreSQL
 * SQL Server
+* Oracle (experimental)
 * MongoDB
+* Redis
 * SQLite
 * Amazon Redshift
 * CockroachDB
@@ -63,13 +65,15 @@ DbGate is licensed under MIT license and is completely free.
   * SQL code completion
   * Add SQL LEFT/INNER/RIGHT join utility
 * Mongo JavaScript editor, execute Mongo script (with NodeJs syntax)
+* Redis tree view, generate script from keys, run Redis script
 * Runs as application for Windows, Linux and Mac. Or in Docker container on server and in web Browser on client.
-* Import, export from/to CSV, Excel, JSON, XML
+* Import, export from/to CSV, Excel, JSON, NDJSON, XML
 * Free table editor - quick table data editing (cleanup data after import/before export, prototype tables etc.)
-* Archives - backup your data in JSON files on local filesystem (or on DbGate server, when using web application)
+* Archives - backup your data in NDJSON files on local filesystem (or on DbGate server, when using web application)
 * Charts, export chart to HTML page
 * For detailed info, how to run DbGate in docker container, visit [docker hub](https://hub.docker.com/r/dbgate/dbgate)
 * Extensible plugin architecture
+* Perspectives - nested table view over complex relational data, query designer on MongoDB databases
 
 ## How to contribute
 Any contributions are welcome. If you want to contribute without coding, consider following:
@@ -77,8 +81,9 @@ Any contributions are welcome. If you want to contribute without coding, conside
 * Tell your friends about DbGate or share on social networks - when more people will use DbGate, it will grow to be better
 * Write review on [Slant.co](https://www.slant.co/improve/options/41086/~dbgate-review) or [G2](https://www.g2.com/products/dbgate/reviews) 
 * Create issue, if you find problem in app, or you have idea to new feature. If issue already exists, you could leave comment on it, to prioritise most wanted issues.
-* Become a backer on [Open collective](https://opencollective.com/dbgate)
-* Where a small coding is acceptable for you, you could [create plugin](https://github.com/dbgate/dbgate/wiki/Plugin-development). Plugins for new themes can be created actually without JS coding.
+* Create some tutorial video on [youtube](https://www.youtube.com/playlist?list=PLCo7KjCVXhr0RfUSjM9wJMsp_ShL1q61A)
+* Become a backer on [GitHub sponsors](https://github.com/sponsors/dbgate) or [Open collective](https://opencollective.com/dbgate)
+* Where a small coding is acceptable for you, you could [create plugin](https://dbgate.org/docs/plugin-development.html). Plugins for new themes can be created actually without JS coding.
 
 Thank you!
 
@@ -91,8 +96,8 @@ There are many database managers now, so why DbGate?
 ## Design goals
 * Application simplicity - DbGate takes the best and only the best from old [DbGate](http://www.jenasoft.com/dbgate), [DatAdmin](http://www.jenasoft.com/datadmin) and [DbMouse](http://www.jenasoft.com/dbmouse) .
 * Minimal dependencies
-    * Frontend - Svelte, socket.io
-    * Backend - NodeJs, ExpressJs, socket.io, database connection drivers
+    * Frontend - Svelte
+    * Backend - NodeJs, ExpressJs, database connection drivers
     * JavaScript + TypeScript
     * App - electron
 * Platform independent - runs as web application in single docker container on server, or as application using Electron platform on Linux, Windows and Mac
@@ -124,11 +129,11 @@ yarn # install NPM packages
 And than run following 3 commands concurrently in 3 terminals:
 ```
 yarn start:api # run API on port 3000
-yarn start:web # run web on port 5000
+yarn start:web # run web on port 5001
 yarn lib # watch typescript libraries and plugins modifications
 ```
-This runs API on port 3000 and web application on port 5000  
-Open http://localhost:5000 in your browser
+This runs API on port 3000 and web application on port 5001  
+Open http://localhost:5001 in your browser
 
 If you want to run electron app:
 ```sh
@@ -139,13 +144,13 @@ yarn # install NPM packages for electron
 
 And than run following 3 commands concurrently in 3 terminals:
 ```
-yarn start:web # run web on port 5000 (only static JS and HTML files)
+yarn start:web # run web on port 5001 (only static JS and HTML files)
 yarn lib # watch typescript libraries and plugins modifications
 yarn start:app # run electron app
 ```
 
 ## How to run built electron app locally
-This mode is very similar to production run of electron app. Electron doesn't use localhost:5000.
+This mode is very similar to production run of electron app. Electron doesn't use localhost:5001.
 
 ```sh
 cd app
@@ -170,4 +175,8 @@ cd dbgate-plugin-my-new-plugin # this directory is created by wizard, edit, what
 yarn plugin # this compiles plugin and copies it into existing DbGate installation
 ```
 
-After restarting DbGate, you could use your new plugin from DbGate.
+After restarting DbGate, you could use your new plugin from DbGate.
+
+## Logging
+DbGate uses [pinomin logger](https://github.com/dbgate/pinomin). So by default, it produces JSON log messages into console and log files. If you want to see formatted logs, please use [pino-pretty](https://github.com/pinojs/pino-pretty) log formatter.
+

adjustPackageJson.js

@@ -0,0 +1,15 @@
+const fs = require('fs');
+
+function adjustFile(file) {
+  const json = JSON.parse(fs.readFileSync(file, { encoding: 'utf-8' }));
+  if (process.platform != 'win32') {
+    delete json.optionalDependencies.msnodesqlv8;
+  }
+  if (process.arch == 'arm64') {
+    delete json.optionalDependencies.oracledb;
+  }
+  fs.writeFileSync(file, JSON.stringify(json, null, 2), 'utf-8');
+}
+
+adjustFile('packages/api/package.json');
+adjustFile('app/package.json');

app/.npmrc

@@ -0,0 +1 @@
+better-sqlite3_local_prebuilds=../../prebuilds

app/entitlements.mac.plist

@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+  <dict>
+    <key>com.apple.security.cs.disable-library-validation</key><true/>
+    <key>com.apple.security.cs.allow-jit</key><true/>
+    <key>com.apple.security.cs.allow-unsigned-executable-memory</key><true/>
+    <key>com.apple.security.cs.allow-dyld-environment-variables</key><true/>
+    <key>com.apple.security.files.user-selected.read-only</key><true/>
+    <key>com.apple.security.files.user-selected.read-write</key><true/>
+  </dict>
+</plist>

app/package.json

@@ -1,6 +1,6 @@
 {
   "name": "dbgate",
-  "version": "4.1.1",
+  "version": "5.0.0-alpha.1",
   "private": true,
   "author": "Jan Prochazka <jenasoft.database@gmail.com>",
   "description": "Opensource database administration tool",
@@ -15,18 +15,23 @@
     "url": "https://github.com/dbgate/dbgate.git"
   },
   "build": {
-    "artifactName": "${productName}-${version}-${os}_${arch}.${ext}",
+    "artifactName": "dbgate-${version}-${os}_${arch}.${ext}",
     "appId": "org.dbgate",
+    "productName": "DbGate",
+    "afterSign": "electron-builder-notarize",
     "mac": {
       "category": "database",
-      "icon": "icon512.png",
+      "icon": "icon512-mac.png",
+      "hardenedRuntime": true,
+      "entitlements": "entitlements.mac.plist",
+      "entitlementsInherit": "entitlements.mac.plist",
       "publish": [
         "github"
       ],
       "target": {
         "target": "default",
         "arch": [
-          "arm64",
+          "universal",
           "x64"
         ]
       }
@@ -66,7 +71,13 @@
     },
     "win": {
       "target": [
-        "nsis",
+        {
+          "target": "nsis",
+          "arch": [
+            "x64",
+            "arm64"
+          ]
+        },
         {
           "target": "zip",
           "arch": [
@@ -89,7 +100,7 @@
   },
   "homepage": "./",
   "scripts": {
-    "start": "cross-env ELECTRON_START_URL=http://localhost:5000 DEVMODE=1 electron .",
+    "start": "cross-env ELECTRON_START_URL=http://localhost:5001 DEVMODE=1 electron .",
     "start:local": "cross-env electron .",
     "dist": "electron-builder",
     "build": "cd ../packages/api && yarn build && cd ../web && yarn build && cd ../../app && yarn dist",
@@ -102,11 +113,13 @@
   "devDependencies": {
     "copyfiles": "^2.2.0",
     "cross-env": "^6.0.3",
-    "electron": "13.6.3",
-    "electron-builder": "22.14.5"
+    "electron": "17.4.10",
+    "electron-builder": "23.1.0",
+    "electron-builder-notarize": "^1.5.0"
   },
   "optionalDependencies": {
-    "better-sqlite3": "7.4.5",
-    "msnodesqlv8": "^2.4.4"
+    "better-sqlite3": "7.6.2",
+    "msnodesqlv8": "^2.6.0",
+    "oracledb": "^5.5.0"
   }
 }

app/src/electron.js

@@ -1,6 +1,8 @@
 const electron = require('electron');
 const os = require('os');
 const fs = require('fs');
+// const unhandled = require('electron-unhandled');
+// const { openNewGitHubIssue, debugInfo } = require('electron-util');
 const { Menu, ipcMain } = require('electron');
 const { autoUpdater } = require('electron-updater');
 const log = require('electron-log');
@@ -14,14 +16,32 @@ const BrowserWindow = electron.BrowserWindow;
 const path = require('path');
 const url = require('url');
 const mainMenuDefinition = require('./mainMenuDefinition');
-
-let useNativeMenu = true;
-let useNativeMenuSpecified = null;
+const { settings } = require('cluster');
 
 // require('@electron/remote/main').initialize();
 
 const configRootPath = path.join(app.getPath('userData'), 'config-root.json');
 let initialConfig = {};
+let apiLoaded = false;
+let mainModule;
+// let getLogger;
+// let loadLogsContent;
+
+const isMac = () => os.platform() == 'darwin';
+
+// unhandled({
+//   showDialog: true,
+//   reportButton: error => {
+//     openNewGitHubIssue({
+//       user: 'dbgate',
+//       repo: 'dbgate',
+//       body: `PLEASE DELETE SENSITIVE INFO BEFORE POSTING ISSUE!!!\n\n\`\`\`\n${
+//         error.stack
+//       }\n\`\`\`\n\n---\n\n${debugInfo()}\n\n\`\`\`\n${loadLogsContent ? loadLogsContent(50) : ''}\n\`\`\``,
+//     });
+//   },
+//   logger: error => (getLogger ? getLogger('electron').fatal(error) : console.error(error)),
+// });
 
 try {
   initialConfig = JSON.parse(fs.readFileSync(configRootPath, { encoding: 'utf-8' }));
@@ -34,6 +54,7 @@ try {
 // be closed automatically when the JavaScript object is garbage collected.
 let mainWindow;
 let mainMenu;
+let runCommandOnLoad = null;
 
 log.transports.file.level = 'debug';
 autoUpdater.logger = log;
@@ -42,119 +63,71 @@ autoUpdater.logger = log;
 
 let commands = {};
 
-function commandItem(id) {
+function formatKeyText(keyText) {
+  if (!keyText) {
+    return keyText;
+  }
+  if (os.platform() == 'darwin') {
+    return keyText.replace('CtrlOrCommand+', 'Command+');
+  }
+  return keyText.replace('CtrlOrCommand+', 'Ctrl+');
+}
+
+function commandItem(item) {
+  const id = item.command;
   const command = commands[id];
+  if (item.skipInApp) {
+    return { skip: true };
+  }
   return {
     id,
     label: command ? command.menuName || command.toolbarName || command.name : id,
-    accelerator: command ? command.keyText : undefined,
+    accelerator: formatKeyText(command ? command.keyText : undefined),
     enabled: command ? command.enabled : false,
     click() {
-      mainWindow.webContents.send('run-command', id);
+      if (mainWindow) {
+        mainWindow.webContents.send('run-command', id);
+      } else {
+        runCommandOnLoad = id;
+        createWindow();
+      }
     },
   };
 }
 
 function buildMenu() {
-  const template = _cloneDeepWith(mainMenuDefinition, item => {
+  let template = _cloneDeepWith(mainMenuDefinition({ editMenu: true }), item => {
     if (item.divider) {
       return { type: 'separator' };
     }
 
     if (item.command) {
-      return commandItem(item.command);
+      return commandItem(item);
     }
   });
-  // const template = [
-  //   {
-  //     label: 'File',
-  //     submenu: [
-  //       commandItem('new.connection'),
-  //       commandItem('new.sqliteDatabase'),
-  //       commandItem('new.modelCompare'),
-  //       commandItem('new.freetable'),
-  //       { type: 'separator' },
-  //       commandItem('file.open'),
-  //       commandItem('file.openArchive'),
-  //       { type: 'separator' },
-  //       commandItem('group.save'),
-  //       commandItem('group.saveAs'),
-  //       commandItem('database.search'),
-  //       { type: 'separator' },
-  //       commandItem('tabs.closeTab'),
-  //       commandItem('file.exit'),
-  //     ],
-  //   },
-  //   {
-  //     label: 'Window',
-  //     submenu: [commandItem('new.query'), { type: 'separator' }, commandItem('tabs.closeAll'), { role: 'minimize' }],
-  //   },
 
-  //   // {
-  //   //   label: 'Edit',
-  //   //   submenu: [
-  //   //     { role: 'undo' },
-  //   //     { role: 'redo' },
-  //   //     { type: 'separator' },
-  //   //     { role: 'cut' },
-  //   //     { role: 'copy' },
-  //   //     { role: 'paste' },
-  //   //   ],
-  //   // },
-  //   {
-  //     label: 'View',
-  //     submenu: [
-  //       { role: 'reload' },
-  //       { role: 'forcereload' },
-  //       { role: 'toggledevtools' },
-  //       { type: 'separator' },
-  //       { role: 'resetzoom' },
-  //       { role: 'zoomin' },
-  //       { role: 'zoomout' },
-  //       { type: 'separator' },
-  //       { role: 'togglefullscreen' },
-  //       commandItem('theme.changeTheme'),
-  //     ],
-  //   },
-  //   {
-  //     role: 'help',
-  //     submenu: [
-  //       {
-  //         label: 'Documentation',
-  //         click() {
-  //           electron.shell.openExternal('https://github.com/dbgate/dbgate/wiki');
-  //         },
-  //       },
-  //       {
-  //         label: 'DbGate web',
-  //         click() {
-  //           electron.shell.openExternal('https://dbgate.org');
-  //         },
-  //       },
-  //       {
-  //         label: 'Report problem or feature request',
-  //         click() {
-  //           electron.shell.openExternal('https://github.com/dbgate/dbgate/issues/new');
-  //         },
-  //       },
-  //       {
-  //         label: 'Become sponsor',
-  //         click() {
-  //           electron.shell.openExternal('https://opencollective.com/dbgate');
-  //         },
-  //       },
-  //       // {
-  //       //   label: 'Discussions',
-  //       //   click() {
-  //       //     electron.shell.openExternal('https://github.com/dbgate/dbgate/discussions');
-  //       //   },
-  //       // },
-  //       { type: 'separator' },
-  //       commandItem('tabs.changelog'),
-  //       commandItem('about.show'),
-  //     ],
-  //   },
-  // ];
+  template = _cloneDeepWith(template, item => {
+    if (Array.isArray(item) && item.find(x => x.skip)) {
+      return item.filter(x => x && !x.skip);
+    }
+  });
+
+  if (isMac()) {
+    template = [
+      {
+        label: 'DbGate',
+        submenu: [
+          commandItem({ command: 'about.show' }),
+          { role: 'services' },
+          { role: 'hide' },
+          { role: 'hideOthers' },
+          { role: 'unhide' },
+          { role: 'quit' },
+        ],
+      },
+      ...template,
+    ];
+  }
 
   return Menu.buildFromTemplate(template);
 }
@@ -169,15 +142,21 @@ ipcMain.on('update-commands', async (event, arg) => {
     // rebuild menu
     if (menu.label != command.text || menu.accelerator != command.keyText) {
       mainMenu = buildMenu();
-      mainWindow.setMenu(mainMenu);
+
+      Menu.setApplicationMenu(mainMenu);
+      // mainWindow.setMenu(mainMenu);
       return;
     }
 
     menu.enabled = command.enabled;
   }
 });
-ipcMain.on('close-window', async (event, arg) => {
-  mainWindow.close();
+ipcMain.on('quit-app', async (event, arg) => {
+  if (isMac()) {
+    app.quit();
+  } else {
+    mainWindow.close();
+  }
 });
 ipcMain.on('set-title', async (event, arg) => {
   mainWindow.setTitle(arg);
@@ -185,10 +164,23 @@ ipcMain.on('set-title', async (event, arg) => {
 ipcMain.on('open-link', async (event, arg) => {
   electron.shell.openExternal(arg);
 });
-ipcMain.on('set-use-native-menu', async (event, arg) => {
-  useNativeMenuSpecified = arg;
+ipcMain.on('open-dev-tools', () => {
+  mainWindow.webContents.openDevTools();
+});
+ipcMain.on('app-started', async (event, arg) => {
+  if (runCommandOnLoad) {
+    mainWindow.webContents.send('run-command', runCommandOnLoad);
+    runCommandOnLoad = null;
+  }
+
+  if (initialConfig['winIsMaximized']) {
+    mainWindow.webContents.send('setIsMaximized', true);
+  }
 });
 ipcMain.on('window-action', async (event, arg) => {
+  if (!mainWindow) {
+    return;
+  }
   switch (arg) {
     case 'minimize':
       mainWindow.minimize();
@@ -196,15 +188,20 @@ ipcMain.on('window-action', async (event, arg) => {
     case 'maximize':
       if (mainWindow.isMaximized()) {
         mainWindow.unmaximize();
+        mainWindow.webContents.send('setIsMaximized', false);
       } else {
         mainWindow.maximize();
+        mainWindow.webContents.send('setIsMaximized', true);
       }
       break;
     case 'close':
       mainWindow.close();
       break;
-    case 'fullscreen':
-      mainWindow.setFullScreen(!mainWindow.isFullScreen());
+    case 'fullscreen-on':
+      mainWindow.setFullScreen(true);
+      break;
+    case 'fullscreen-off':
+      mainWindow.setFullScreen(false);
       break;
     case 'devtools':
       mainWindow.webContents.toggleDevTools();
@@ -212,6 +209,32 @@ ipcMain.on('window-action', async (event, arg) => {
     case 'reload':
       mainWindow.webContents.reloadIgnoringCache();
       break;
+    case 'zoomin':
+      mainWindow.webContents.zoomLevel += 0.5;
+      break;
+    case 'zoomout':
+      mainWindow.webContents.zoomLevel -= 0.5;
+      break;
+    case 'zoomreset':
+      mainWindow.webContents.zoomLevel = 0;
+      break;
+
+    // edit
+    case 'undo':
+      mainWindow.webContents.undo();
+      break;
+    case 'redo':
+      mainWindow.webContents.redo();
+      break;
+    case 'cut':
+      mainWindow.webContents.cut();
+      break;
+    case 'copy':
+      mainWindow.webContents.copy();
+      break;
+    case 'paste':
+      mainWindow.webContents.paste();
+      break;
   }
 });
 
@@ -229,21 +252,32 @@ ipcMain.handle('showItemInFolder', async (event, path) => {
 ipcMain.handle('openExternal', async (event, url) => {
   electron.shell.openExternal(url);
 });
-ipcMain.handle('useNativeMenu', async () => {
-  return useNativeMenu;
-});
+
+function fillMissingSettings(value) {
+  const res = {
+    ...value,
+  };
+  if (value['app.useNativeMenu'] !== true && value['app.useNativeMenu'] !== false) {
+    res['app.useNativeMenu'] = false;
+    // res['app.useNativeMenu'] = os.platform() == 'darwin' ? true : false;
+  }
+  return res;
+}
 
 function createWindow() {
+  let settingsJson = {};
+  try {
+    const datadir = path.join(os.homedir(), '.dbgate');
+    settingsJson = fillMissingSettings(
+      JSON.parse(fs.readFileSync(path.join(datadir, 'settings.json'), { encoding: 'utf-8' }))
+    );
+  } catch (err) {
+    console.log('Error loading settings.json:', err.message);
+    settingsJson = fillMissingSettings({});
+  }
+
   const bounds = initialConfig['winBounds'];
-  useNativeMenu = os.platform() == 'darwin' ? true : false;
-  if (initialConfig['useNativeMenu'] === true) {
-    useNativeMenu = true;
-    useNativeMenuSpecified = true;
-  }
-  if (initialConfig['useNativeMenu'] === false) {
-    useNativeMenu = false;
-    useNativeMenuSpecified = false;
-  }
+  useNativeMenu = settingsJson['app.useNativeMenu'];
 
   mainWindow = new BrowserWindow({
     width: 1200,
@@ -253,6 +287,7 @@ function createWindow() {
     titleBarStyle: useNativeMenu ? undefined : 'hidden',
     ...bounds,
     icon: os.platform() == 'win32' ? 'icon.ico' : path.resolve(__dirname, '../icon.png'),
+    partition: 'persist:dbgate',
     webPreferences: {
       nodeIntegration: true,
       contextIsolation: false,
@@ -263,6 +298,9 @@ function createWindow() {
   if (initialConfig['winIsMaximized']) {
     mainWindow.maximize();
   }
+  if (settingsJson['app.fullscreen']) {
+    mainWindow.setFullScreen(true);
+  }
 
   mainMenu = buildMenu();
   mainWindow.setMenu(mainMenu);
@@ -282,7 +320,6 @@ function createWindow() {
           JSON.stringify({
             winBounds: mainWindow.getBounds(),
             winIsMaximized: mainWindow.isMaximized(),
-            useNativeMenu: useNativeMenuSpecified,
           }),
           'utf-8'
         );
@@ -294,27 +331,35 @@ function createWindow() {
     if (os.platform() == 'linux') {
       mainWindow.setIcon(path.resolve(__dirname, '../icon.png'));
     }
+    // mainWindow.webContents.toggleDevTools();
   }
 
-  const apiPackage = path.join(
-    __dirname,
-    process.env.DEVMODE ? '../../packages/api/src/index' : '../packages/api/dist/bundle.js'
-  );
+  if (!apiLoaded) {
+    const apiPackage = path.join(
+      __dirname,
+      process.env.DEVMODE ? '../../packages/api/src/index' : '../packages/api/dist/bundle.js'
+    );
 
-  global.API_PACKAGE = apiPackage;
-  global.NATIVE_MODULES = path.join(__dirname, 'nativeModules');
+    global.API_PACKAGE = apiPackage;
+    global.NATIVE_MODULES = path.join(__dirname, 'nativeModules');
 
-  // console.log('global.API_PACKAGE', global.API_PACKAGE);
-  const api = require(apiPackage);
-  // console.log(
-  //   'REQUIRED',
-  //   path.resolve(
-  //     path.join(__dirname, process.env.DEVMODE ? '../../packages/api/src/index' : '../packages/api/dist/bundle.js')
-  //   )
-  // );
-  const main = api.getMainModule();
-  main.initializeElectronSender(mainWindow.webContents);
-  main.useAllControllers(null, electron);
+    // console.log('global.API_PACKAGE', global.API_PACKAGE);
+    const api = require(apiPackage);
+    // console.log(
+    //   'REQUIRED',
+    //   path.resolve(
+    //     path.join(__dirname, process.env.DEVMODE ? '../../packages/api/src/index' : '../packages/api/dist/bundle.js')
+    //   )
+    // );
+    api.configureLogger();
+    const main = api.getMainModule();
+    main.useAllControllers(null, electron);
+    mainModule = main;
+    // getLogger = api.getLogger;
+    // loadLogsContent = api.loadLogsContent;
+    apiLoaded = true;
+  }
+  mainModule.setElectronSender(mainWindow.webContents);
 
   loadMainWindow();
 
@@ -324,6 +369,7 @@ function createWindow() {
     // in an array if your app supports multi windows, this is the time
     // when you should delete the corresponding element.
     mainWindow = null;
+    mainModule.setElectronSender(null);
   });
 }
 
@@ -343,7 +389,7 @@ app.on('ready', onAppReady);
 app.on('window-all-closed', function () {
   // On OS X it is common for applications and their menu bar
   // to stay active until the user quits explicitly with Cmd + Q
-  if (process.platform !== 'darwin') {
+  if (!isMac()) {
     app.quit();
   }
 });

app/src/mainMenuDefinition.js

@@ -1,4 +1,4 @@
-module.exports = [
+module.exports = ({ editMenu }) => [
   {
     label: 'File',
     submenu: [
@@ -6,6 +6,9 @@ module.exports = [
       { command: 'new.sqliteDatabase', hideDisabled: true },
       { divider: true },
       { command: 'new.query', hideDisabled: true },
+      { command: 'new.queryDesign', hideDisabled: true },
+      { command: 'new.diagram', hideDisabled: true },
+      { command: 'new.perspective', hideDisabled: true },
       { command: 'new.freetable', hideDisabled: true },
       { command: 'new.shell', hideDisabled: true },
       { command: 'new.jsonl', hideDisabled: true },
@@ -17,16 +20,38 @@ module.exports = [
       { command: 'group.saveAs', hideDisabled: true },
       { divider: true },
       { command: 'file.exit', hideDisabled: true },
+      { command: 'app.logout', hideDisabled: true, skipInApp: true },
+      { command: 'app.disconnect', hideDisabled: true, skipInApp: true },
     ],
   },
   {
     label: 'Window',
     submenu: [
-      { command: 'tabs.closeTab', hideDisabled: true },
-      { command: 'tabs.closeAll', hideDisabled: true },
+      { command: 'tabs.closeTab', hideDisabled: false },
+      { command: 'tabs.closeAll', hideDisabled: false },
+      { command: 'tabs.closeTabsWithCurrentDb', hideDisabled: false },
+      { command: 'tabs.closeTabsButCurrentDb', hideDisabled: false },
+      { divider: true },
+      { command: 'app.zoomIn', hideDisabled: true },
+      { command: 'app.zoomOut', hideDisabled: true },
+      { command: 'app.zoomReset', hideDisabled: true },
     ],
   },
 
+  editMenu
+    ? {
+        label: 'Edit',
+        submenu: [
+          { command: 'edit.undo' },
+          { command: 'edit.redo' },
+          { divider: true },
+          { command: 'edit.cut' },
+          { command: 'edit.copy' },
+          { command: 'edit.paste' },
+        ],
+      }
+    : null,
+
   // {
   //   label: 'Edit',
   //   submenu: [
@@ -45,6 +70,7 @@ module.exports = [
       { command: 'app.toggleDevTools', hideDisabled: true },
       { command: 'app.toggleFullScreen', hideDisabled: true },
       { command: 'app.minimize', hideDisabled: true },
+      { command: 'toggle.sidebar' },
       { divider: true },
       { command: 'theme.changeTheme', hideDisabled: true },
       { command: 'settings.show' },
@@ -55,10 +81,14 @@ module.exports = [
     submenu: [
       { command: 'database.search', hideDisabled: true },
       { command: 'commandPalette.show', hideDisabled: true },
+      { command: 'database.switch', hideDisabled: true },
       { divider: true },
       { command: 'sql.generator', hideDisabled: true },
       { command: 'file.import', hideDisabled: true },
       { command: 'new.modelCompare', hideDisabled: true },
+      { divider: true },
+      { command: 'folder.showLogs', hideDisabled: true },
+      { command: 'folder.showData', hideDisabled: true },
     ],
   },
   {
@@ -69,6 +99,7 @@ module.exports = [
       { command: 'app.openIssue', hideDisabled: true },
       { command: 'app.openSponsoring', hideDisabled: true },
       { divider: true },
+      { command: 'settings.commands', hideDisabled: true },
       { command: 'tabs.changelog', hideDisabled: true },
       { command: 'about.show', hideDisabled: true },
     ],

docker-compose.yaml

@@ -5,6 +5,8 @@ services:
   dbgate:
     build: docker
     # image: dbgate/dbgate:beta-alpine
+    # image: dbgate/dbgate:alpine
+    # image: dbgate/dbgate:beta
     restart: always
     ports:
       - 3100:3000
@@ -12,11 +14,24 @@ services:
     #   - /home/jena/dbgate-data:/root/dbgate-data
 
     volumes:
-      - dbgate-data:/root/dbgate-data
+      - dbgate-data:/root/.dbgate
     
     # environment:
     #   WEB_ROOT: /dbgate
 
+      # CONNECTIONS: mssql
+      # LABEL_mssql: MS Sql
+      # SERVER_mssql: mssql
+      # USER_mssql: sa
+      # PORT_mssql: 1433
+      # PASSWORD_mssql: Pwd2020Db
+      # ENGINE_mssql: mssql@dbgate-plugin-mssql
+  # proxy:
+  #   # image: nginx
+  #   build: test/nginx
+  #   ports:
+  #     - 8082:80
+
     # volumes:
     #   - /home/jena/test/chinook:/mnt/sqt
     # environment:
@@ -26,6 +41,14 @@ services:
     #   FILE_sqlite: /mnt/sqt/Chinook.db
     #   ENGINE_sqlite: sqlite@dbgate-plugin-sqlite
 
+  # mssql:
+  #   image: mcr.microsoft.com/mssql/server
+  #   restart: always
+  #   environment:
+  #     - ACCEPT_EULA=Y
+  #     - SA_PASSWORD=Pwd2020Db
+  #     - MSSQL_PID=Express
+
 volumes:
   dbgate-data:
     driver: local

docker/Dockerfile

@@ -1,10 +1,18 @@
 FROM node:14
 
+RUN apt-get update && apt-get install -y \
+    iputils-ping \
+    iproute2 \
+ && rm -rf /var/lib/apt/lists/*
+
 WORKDIR /home/dbgate-docker
 
 COPY . .
 
+RUN ["chmod", "+x", "/home/dbgate-docker/entrypoint.sh"]
+
 WORKDIR /home/dbgate-docker
 EXPOSE 3000
-VOLUME /root/dbgate-data
-CMD node bundle.js
+VOLUME /root/.dbgate
+
+CMD ["/home/dbgate-docker/entrypoint.sh"]

docker/Dockerfile-alpine

@@ -2,9 +2,16 @@ FROM node:14-alpine
 
 WORKDIR /home/dbgate-docker
 
+RUN apk --no-cache upgrade \
+ && apk --no-cache add  \
+    iputils
+
 COPY . .
 
+RUN ["chmod", "+x", "/home/dbgate-docker/entrypoint.sh"]
+
 WORKDIR /home/dbgate-docker
 EXPOSE 3000
-VOLUME /root/dbgate-data
-CMD node bundle.js
+VOLUME /root/.dbgate
+
+CMD ["/home/dbgate-docker/entrypoint.sh"]

docker/entrypoint.sh

@@ -0,0 +1,11 @@
+#!/bin/sh
+
+HOST_DOMAIN="dockerhost"
+ping -q -c1 $HOST_DOMAIN > /dev/null 2>&1
+if [ $? != 0 ]
+then
+  HOST_IP=$(ip route | awk 'NR==1 {print $3}')
+  echo "$HOST_IP $HOST_DOMAIN" >> /etc/hosts
+fi
+
+node bundle.js --listen-api

fillNativeModules.js

@@ -5,9 +5,12 @@ let fillContent = '';
 if (process.platform == 'win32') {
   fillContent += `content.msnodesqlv8 = () => require('msnodesqlv8');`;
 }
+if (process.arch != 'arm64') {
+  fillContent += `content.oracledb = () => require('oracledb');`;
+}
 fillContent += `content['better-sqlite3'] = () => require('better-sqlite3');`;
 
-const getContent = (empty) => `
+const getContent = empty => `
 // this file is generated automatically by script fillNativeModules.js, do not edit it manually
 const content = {};
 

integration-tests/__tests__/data-duplicator.spec.js

@@ -0,0 +1,94 @@
+const engines = require('../engines');
+const stream = require('stream');
+const { testWrapper } = require('../tools');
+const dataDuplicator = require('dbgate-api/src/shell/dataDuplicator');
+const { runCommandOnDriver } = require('dbgate-tools');
+
+describe('Data duplicator', () => {
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Insert simple data - %s',
+    testWrapper(async (conn, driver, engine) => {
+      runCommandOnDriver(conn, driver, dmp =>
+        dmp.createTable({
+          pureName: 't1',
+          columns: [
+            { columnName: 'id', dataType: 'int', autoIncrement: true, notNull: true },
+            { columnName: 'val', dataType: 'varchar(50)' },
+          ],
+          primaryKey: {
+            columns: [{ columnName: 'id' }],
+          },
+        })
+      );
+      runCommandOnDriver(conn, driver, dmp =>
+        dmp.createTable({
+          pureName: 't2',
+          columns: [
+            { columnName: 'id', dataType: 'int', autoIncrement: true, notNull: true },
+            { columnName: 'val', dataType: 'varchar(50)' },
+            { columnName: 'valfk', dataType: 'int', notNull: true },
+          ],
+          primaryKey: {
+            columns: [{ columnName: 'id' }],
+          },
+          foreignKeys: [{ refTableName: 't1', columns: [{ columnName: 'valfk', refColumnName: 'id' }] }],
+        })
+      );
+
+      const gett1 = () =>
+        stream.Readable.from([
+          { __isStreamHeader: true, __isDynamicStructure: true },
+          { id: 1, val: 'v1' },
+          { id: 2, val: 'v2' },
+          { id: 3, val: 'v3' },
+        ]);
+      const gett2 = () =>
+        stream.Readable.from([
+          { __isStreamHeader: true, __isDynamicStructure: true },
+          { id: 1, val: 'v1', valfk: 1 },
+          { id: 2, val: 'v2', valfk: 2 },
+          { id: 3, val: 'v3', valfk: 3 },
+        ]);
+
+      await dataDuplicator({
+        systemConnection: conn,
+        driver,
+        items: [
+          {
+            name: 't1',
+            operation: 'copy',
+            openStream: gett1,
+          },
+          {
+            name: 't2',
+            operation: 'copy',
+            openStream: gett2,
+          },
+        ],
+      });
+
+      await dataDuplicator({
+        systemConnection: conn,
+        driver,
+        items: [
+          {
+            name: 't1',
+            operation: 'copy',
+            openStream: gett1,
+          },
+          {
+            name: 't2',
+            operation: 'copy',
+            openStream: gett2,
+          },
+        ],
+      });
+
+      const res1 = await driver.query(conn, `select count(*) as cnt from t1`);
+      expect(res1.rows[0].cnt.toString()).toEqual('6');
+
+      const res2 = await driver.query(conn, `select count(*) as cnt from t2`);
+      expect(res2.rows[0].cnt.toString()).toEqual('6');
+    })
+  );
+});

integration-tests/__tests__/deploy-database.spec.js

@@ -297,4 +297,33 @@ describe('Deploy database', () => {
       expect(res.rows[0].val.toString()).toEqual('5');
     })
   );
+
+  test.each(engines.enginesPostgre.map(engine => [engine.label, engine]))(
+    'Current timestamp default value - %s',
+    testWrapper(async (conn, driver, engine) => {
+      await testDatabaseDeploy(conn, driver, [
+        [
+          {
+            name: 't1.table.yaml',
+            json: {
+              name: 't1',
+              columns: [
+                { name: 'id', type: 'int' },
+                {
+                  name: 'val',
+                  type: 'timestamp',
+                  default: 'current_timestamp',
+                },
+              ],
+              primaryKey: ['id'],
+            },
+          },
+        ],
+      ]);
+
+      await driver.query(conn, `insert into t1 (id) values (1)`);
+      const res = await driver.query(conn, ` select val from t1 where id = 1`);
+      expect(res.rows[0].val.toString().substring(0, 2)).toEqual('20');
+    })
+  );
 });

integration-tests/__tests__/table-analyse.spec.js

@@ -14,7 +14,7 @@ const txMatch = (tname, vcolname, nextcol) =>
       expect.objectContaining({
         columnName: 'id',
         notNull: true,
-        dataType: expect.stringContaining('int'),
+        dataType: expect.stringMatching(/int/i),
       }),
       expect.objectContaining({
         columnName: vcolname,

integration-tests/docker-compose.yaml

@@ -1,21 +1,30 @@
 version: '3'
 services:
-  postgres:
-    image: postgres
-    restart: always
-    environment:
-      POSTGRES_PASSWORD: Pwd2020Db
-    ports: 
-      - 15000:5432
+  # postgres:
+  #   image: postgres
+  #   restart: always
+  #   environment:
+  #     POSTGRES_PASSWORD: Pwd2020Db
+  #   ports: 
+  #     - 15000:5432
 
-  mysql:
-    image: mysql:8.0.18
-    command: --default-authentication-plugin=mysql_native_password
-    restart: always
-    ports: 
-      - 15001:3306
-    environment:
-      - MYSQL_ROOT_PASSWORD=Pwd2020Db
+  # mariadb:
+  #   image: mariadb
+  #   command: --default-authentication-plugin=mysql_native_password
+  #   restart: always
+  #   ports: 
+  #     - 15004:3306
+  #   environment:
+  #     - MYSQL_ROOT_PASSWORD=Pwd2020Db
+
+  # mysql:
+  #   image: mysql:8.0.18
+  #   command: --default-authentication-plugin=mysql_native_password
+  #   restart: always
+  #   ports: 
+  #     - 15001:3306
+  #   environment:
+  #     - MYSQL_ROOT_PASSWORD=Pwd2020Db
 
   mssql:
     image: mcr.microsoft.com/mssql/server
@@ -27,11 +36,11 @@ services:
       - SA_PASSWORD=Pwd2020Db
       - MSSQL_PID=Express
       
-  cockroachdb:
-    image: cockroachdb/cockroach
-    ports:
-      - 15003:26257
-    command: start-single-node --insecure
+  # cockroachdb:
+  #   image: cockroachdb/cockroach
+  #   ports:
+  #     - 15003:26257
+  #   command: start-single-node --insecure
 
 #   mongodb:
 #     image: mongo:4.0.12

integration-tests/engines.js

@@ -31,6 +31,23 @@ const engines = [
     objects: [views],
     dbSnapshotBySeconds: true,
   },
+  {
+    label: 'MariaDB',
+    connection: {
+      engine: 'mariadb@dbgate-plugin-mysql',
+      password: 'Pwd2020Db',
+      user: 'root',
+      server: 'mysql',
+      port: 3306,
+    },
+    local: {
+      server: 'localhost',
+      port: 15004,
+    },
+    skipOnCI: true,
+    objects: [views],
+    dbSnapshotBySeconds: true,
+  },
   {
     label: 'PostgreSQL',
     connection: {
@@ -117,12 +134,17 @@ const engines = [
 const filterLocal = [
   // filter local testing
   '-MySQL',
+  '-MariaDB',
   '-PostgreSQL',
   '-SQL Server',
   'SQLite',
   '-CockroachDB',
 ];
 
+const enginesPostgre = engines.filter(x => x.label == 'PostgreSQL');
+
 module.exports = process.env.CITEST
   ? engines.filter(x => !x.skipOnCI)
   : engines.filter(x => filterLocal.find(y => x.label == y));
+
+module.exports.enginesPostgre = enginesPostgre;

integration-tests/package.json

@@ -1,6 +1,6 @@
 {
   "name": "dbgate-integration-tests",
-  "version": "4.1.1",
+  "version": "5.0.0-alpha.1",
   "homepage": "https://dbgate.org/",
   "repository": {
     "type": "git",
@@ -13,6 +13,8 @@
     "wait:ci": "cross-env DEVMODE=1 CITEST=1 node wait.js",
 
     "test:local": "cross-env DEVMODE=1 LOCALTEST=1 jest",
+    "test:local:path": "cross-env DEVMODE=1 LOCALTEST=1 jest --runTestsByPath __tests__/data-duplicator.spec.js",
+    
     "test:ci": "cross-env DEVMODE=1 CITEST=1 jest --runInBand --json --outputFile=result.json --testLocationInResults",
     
     "run:local": "docker-compose down && docker-compose up -d && yarn wait:local && yarn test:local"

misc/convert-icons.sh

@@ -0,0 +1,109 @@
+# magick icon.svg -resize 1024x1024 -transparent white -background transparent app/icon1024.png
+
+# magick app/icon1024.png -resize 512x512 app/icon512.png
+# magick app/icon1024.png -resize 256x256 app/icon.png
+# magick app/icon1024.png -resize 32x32 app/icon32.png
+# magick icon.svg -define icon:auto-resize="256,128,96,64,48,32,16" -transparent white -background transparent app/icon.ico
+
+# # magick icon.svg -resize 512x512 -transparent white -background transparent app/icon512.png
+# # magick icon.svg -resize 256x256 -transparent white -background transparent app/icon.png
+# # magick icon.svg -resize 32x32 -transparent white -background transparent app/icon32.png
+# # magick icon.svg -define icon:auto-resize="256,128,96,64,48,32,16" -transparent white -background transparent app/icon.ico
+
+# magick app/icon1024.png -resize 512x512 app/icons/512x512.png
+# magick app/icon1024.png -resize 256x256 app/icons/256x256.png
+# magick app/icon1024.png -resize 128x128 app/icons/128x128.png
+# magick app/icon1024.png -resize 64x64 app/icons/64x64.png
+# magick app/icon1024.png -resize 48x48 app/icons/48x48.png
+# magick app/icon1024.png -resize 32x32 app/icons/32x32.png
+# magick app/icon1024.png -resize 16x16 app/icons/16x16.png
+
+# # magick icon.svg -resize 16x16 -transparent white -background transparent app/icons/16x16.png
+# # magick icon.svg -resize 32x32 -transparent white -background transparent app/icons/32x32.png
+# # magick icon.svg -resize 48x48 -transparent white -background transparent app/icons/48x48.png
+# # magick icon.svg -resize 64x64 -transparent white -background transparent app/icons/64x64.png
+# # magick icon.svg -resize 128x128 -transparent white -background transparent app/icons/128x128.png
+# # magick icon.svg -resize 256x256 -transparent white -background transparent app/icons/256x256.png
+# # magick icon.svg -resize 512x512 -transparent white -background transparent app/icons/512x512.png
+
+# magick icon.svg -resize 1024x1024 icon.png
+# magick icon.svg -resize 1024x1024 -transparent white -background transparent icon.png
+
+STROKE_WIDTH=30
+LEFT=150
+RIGHT=850
+
+
+
+magick  \
+    \( \
+        -size 1000x1000 -define gradient:direction=east 'gradient:#0050b3-#1890ff' \
+        \( +clone -fill Black -colorize 100 \
+        -fill White -stroke White -draw "arc $LEFT,750 $RIGHT,950 0,360" -draw "rectangle $LEFT,150 $RIGHT,850" \
+        \) \
+        -alpha off \
+        -compose CopyOpacity -composite \
+    \) \
+    \( \
+        -size 1000x1000 -define gradient:direction=east 'gradient:#096dd9-#40a9ff' \
+        \( +clone -fill Black -colorize 100 \
+        -fill White -draw "arc $LEFT,50 $RIGHT,250 0,360" \
+        \) \
+        -alpha off \
+        -compose CopyOpacity -composite \
+    \) \
+    -compose Over -composite \
+    -strokewidth $STROKE_WIDTH -stroke '#0050b3' -fill transparent \
+    -draw "arc $LEFT,225 $RIGHT,425  0,180" \
+    -draw "arc $LEFT,400 $RIGHT,600  0,180" \
+    -draw "arc $LEFT,575 $RIGHT,775  0,180" \
+    -draw "arc $LEFT,750 $RIGHT,950  0,180" \
+    -draw "arc $LEFT,50 $RIGHT,250  0,360" \
+    -draw "line $LEFT,150 $LEFT,850" \
+    -draw "line $RIGHT,150 $RIGHT,850" \
+    -fill '#fafafa' -stroke '#8c8c8c' -strokewidth 3 \
+    -pointsize 800 -font './Mcbungus-Regular.ttf' \
+    -gravity center \
+    -draw 'text 0,100 "G"' \
+    icon.png
+
+
+# magick \
+#   \( \
+#     -size 300x300 gradient:red-blue \
+#     \( +clone -fill Black -colorize 100 \
+#       -fill White -draw "polygon 50,50 250,50 200,200" \
+#     \) \
+#     -alpha off \
+#     -compose CopyOpacity -composite \
+#   \) \
+#   \( \
+#     -size 300x300 'gradient:#f80-#08f' \
+#     \( +clone -fill Black -colorize 100 \
+#       -fill White -draw "polygon 50,150 250,150 200,300" \
+#     \) \
+#     -alpha off \
+#     -compose CopyOpacity -composite \
+#   \) \
+#   -compose Over -composite \
+#   icon.png    
+
+magick icon.png -resize 512x512! ../app/icon512.png
+magick icon.png -resize 256x256! ../app/icon.png
+magick icon.png -resize 32x32! ../app/icon32.png
+magick icon.png -define icon:auto-resize="256,128,96,64,48,32,16" ../app/icon.ico
+
+magick icon.png -resize 512x512! ../app/icons/512x512.png
+magick icon.png -resize 256x256! ../app/icons/256x256.png
+magick icon.png -resize 128x128! ../app/icons/128x128.png
+magick icon.png -resize 64x64! ../app/icons/64x64.png
+magick icon.png -resize 48x48! ../app/icons/48x48.png
+magick icon.png -resize 32x32! ../app/icons/32x32.png
+magick icon.png -resize 16x16! ../app/icons/16x16.png
+
+magick icon.png -resize 192x192! ../packages/web/public/logo192.png
+magick icon.png -resize 512x512! ../packages/web/public/logo512.png
+magick icon.png -define icon:auto-resize="256,128,96,64,48,32,16" ../packages/web/public/favicon.ico
+
+convert icon.png -resize 800x800 -background transparent -gravity center -extent 1000x1000 iconmac.png
+magick composite  iconmac.png macbg.png -resize 600x600! ../app/icon512-mac.png

misc/play-dark-mode.svg

@@ -0,0 +1,14 @@
+<?xml version="1.0" encoding="iso-8859-1"?>
+<!-- Generator: Adobe Illustrator 18.1.1, SVG Export Plug-In . SVG Version: 6.00 Build 0)  -->
+<svg version="1.1" id="Capa_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px"
+	 viewBox="0 0 17.804 17.804" style="enable-background:new 0 0 17.804 17.804;" xml:space="preserve">
+<g>
+	<g id="c98_play">
+		<path fill='#ccc' d="M2.067,0.043C2.21-0.028,2.372-0.008,2.493,0.085l13.312,8.503c0.094,0.078,0.154,0.191,0.154,0.313
+			c0,0.12-0.061,0.237-0.154,0.314L2.492,17.717c-0.07,0.057-0.162,0.087-0.25,0.087l-0.176-0.04
+			c-0.136-0.065-0.222-0.207-0.222-0.361V0.402C1.844,0.25,1.93,0.107,2.067,0.043z"/>
+	</g>
+	<g id="Capa_1_78_">
+	</g>
+</g>
+</svg>

misc/play-light-mode.svg

@@ -0,0 +1,14 @@
+<?xml version="1.0" encoding="iso-8859-1"?>
+<!-- Generator: Adobe Illustrator 18.1.1, SVG Export Plug-In . SVG Version: 6.00 Build 0)  -->
+<svg version="1.1" id="Capa_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px"
+	 viewBox="0 0 17.804 17.804" style="enable-background:new 0 0 17.804 17.804;" xml:space="preserve">
+<g>
+	<g id="c98_play">
+		<path fill='#444' d="M2.067,0.043C2.21-0.028,2.372-0.008,2.493,0.085l13.312,8.503c0.094,0.078,0.154,0.191,0.154,0.313
+			c0,0.12-0.061,0.237-0.154,0.314L2.492,17.717c-0.07,0.057-0.162,0.087-0.25,0.087l-0.176-0.04
+			c-0.136-0.065-0.222-0.207-0.222-0.361V0.402C1.844,0.25,1.93,0.107,2.067,0.043z"/>
+	</g>
+	<g id="Capa_1_78_">
+	</g>
+</g>
+</svg>

package.json

@@ -1,6 +1,6 @@
 {
   "private": true,
-  "version": "4.7.1-docker.2",
+  "version": "5.2.5-beta.16",
   "name": "dbgate-all",
   "workspaces": [
     "packages/*",
@@ -8,22 +8,28 @@
     "integration-tests"
   ],
   "scripts": {
-    "start:api": "yarn workspace dbgate-api start",
-    "start:app": "cd app && yarn start",
-    "start:api:portal": "yarn workspace dbgate-api start:portal",
-    "start:api:covid": "yarn workspace dbgate-api start:covid",
+    "start:api": "yarn workspace dbgate-api start | pino-pretty",
+    "start:api:json": "yarn workspace dbgate-api start",
+    "start:app": "cd app && yarn start | pino-pretty",
+    "start:app:singledb": "CONNECTIONS=con1 SERVER_con1=localhost ENGINE_con1=mysql@dbgate-plugin-mysql USER_con1=root PASSWORD_con1=Pwd2020Db SINGLE_CONNECTION=con1 SINGLE_DATABASE=Chinook yarn start:app",
+    "start:api:debug": "cross-env DEBUG=* yarn workspace dbgate-api start",
+    "start:app:debug": "cd app && cross-env DEBUG=* yarn start",
+    "start:api:debug:ssh": "cross-env DEBUG=ssh yarn workspace dbgate-api start",
+    "start:app:debug:ssh": "cd app && cross-env DEBUG=ssh yarn start",
+    "start:api:portal": "yarn workspace dbgate-api start:portal | pino-pretty",
+    "start:api:singledb": "yarn workspace dbgate-api start:singledb | pino-pretty",
+    "start:api:auth": "yarn workspace dbgate-api start:auth | pino-pretty",
+    "start:api:dblogin": "yarn workspace dbgate-api start:dblogin | pino-pretty",
     "start:web": "yarn workspace dbgate-web dev",
     "start:sqltree": "yarn workspace dbgate-sqltree start",
     "start:tools": "yarn workspace dbgate-tools start",
     "start:datalib": "yarn workspace dbgate-datalib start",
     "start:filterparser": "yarn workspace dbgate-filterparser start",
-    "start:querysplitter": "yarn workspace dbgate-query-splitter start",
     "build:sqltree": "yarn workspace dbgate-sqltree build",
     "build:datalib": "yarn workspace dbgate-datalib build",
     "build:filterparser": "yarn workspace dbgate-filterparser build",
-    "build:querysplitter": "yarn workspace dbgate-query-splitter build",
     "build:tools": "yarn workspace dbgate-tools build",
-    "build:lib": "yarn build:querysplitter && yarn build:tools && yarn build:sqltree && yarn build:filterparser && yarn build:datalib",
+    "build:lib": "yarn build:sqltree && yarn build:tools && yarn build:filterparser && yarn build:datalib",
     "build:app": "yarn plugins:copydist && cd app && yarn install && yarn build",
     "build:api": "yarn workspace dbgate-api build",
     "build:web:docker": "yarn workspace dbgate-web build",
@@ -34,6 +40,7 @@
     "start:app:local": "cd app && yarn start:local",
     "setCurrentVersion": "node setCurrentVersion",
     "generatePadFile": "node generatePadFile",
+    "adjustPackageJson": "node adjustPackageJson",
     "fillNativeModules": "node fillNativeModules",
     "fillNativeModulesElectron": "node fillNativeModules --electron",
     "fillPackagedPlugins": "node fillPackagedPlugins",
@@ -43,16 +50,17 @@
     "install:sqlite:docker": "cd docker && yarn init --yes && yarn add better-sqlite3 && cd ..",
     "prepare:docker": "yarn plugins:copydist && yarn build:web:docker && yarn build:api && yarn copy:docker:build && yarn install:sqlite:docker",
     "start": "concurrently --kill-others-on-fail \"yarn start:api\" \"yarn start:web\"",
-    "lib": "concurrently --kill-others-on-fail \"yarn start:sqltree\" \"yarn start:filterparser\" \"yarn start:datalib\" \"yarn start:tools\" \"yarn start:querysplitter\" \"yarn build:plugins:frontend:watch\"",
+    "lib": "concurrently --kill-others-on-fail \"yarn start:sqltree\" \"yarn start:filterparser\" \"yarn start:datalib\" \"yarn start:tools\" \"yarn build:plugins:frontend:watch\"",
     "ts:api": "yarn workspace dbgate-api ts",
     "ts:web": "yarn workspace dbgate-web ts",
     "ts": "yarn ts:api && yarn ts:web",
-    "postinstall": "yarn resetPackagedPlugins && yarn build:lib && patch-package && yarn fillNativeModules && yarn build:plugins:frontend"
+    "postinstall": "yarn resetPackagedPlugins && yarn build:lib && patch-package && yarn fillNativeModules && yarn build:plugins:frontend",
+    "dbgate-serve": "node packages/dbgate/bin/dbgate-serve.js"
   },
   "dependencies": {
     "concurrently": "^5.1.0",
     "patch-package": "^6.2.1",
-    "socket.io": "^2.3.0"
+    "pino-pretty": "^9.1.1"
   },
   "devDependencies": {
     "copyfiles": "^2.2.0",

packages/api/.env

@@ -1 +1,17 @@
 DEVMODE=1
+SHELL_SCRIPTING=1
+
+# PERMISSIONS=~widgets/app,~widgets/plugins
+# DISABLE_SHELL=1
+# HIDE_APP_EDITOR=1
+
+
+# DEVWEB=1
+# LOGINS=admin,test
+
+# LOGIN_PASSWORD_admin=admin
+# LOGIN_PERMISSIONS_admin=*
+
+# LOGIN_PASSWORD_test=test
+# LOGIN_PERMISSIONS_test=~*, widgets/database
+# WORKSPACE_DIR=/home/jena/dbgate-data-2

packages/api/env/auth/.gitignore

@@ -0,0 +1 @@
+.env

packages/api/env/dblogin/.env

@@ -0,0 +1,14 @@
+DEVMODE=1
+
+CONNECTIONS=mysql
+SINGLE_CONNECTION=mysql
+# SINGLE_DATABASE=Chinook
+
+LABEL_mysql=MySql localhost
+SERVER_mysql=localhost
+# USER_mysql=root
+PORT_mysql=3306
+# PASSWORD_mysql=Pwd2020Db
+ENGINE_mysql=mysql@dbgate-plugin-mysql
+# PASSWORD_MODE_mysql=askPassword
+PASSWORD_MODE_mysql=askUser

packages/api/env/portal/.env

@@ -1,6 +1,6 @@
 DEVMODE=1
 
-CONNECTIONS=mysql,postgres,mongo,mongo2,mysqlssh,sqlite
+CONNECTIONS=mysql,postgres,mongo,mongo2,mysqlssh,sqlite,relational
 
 LABEL_mysql=MySql localhost
 SERVER_mysql=localhost
@@ -38,7 +38,25 @@ SSH_LOGIN_mysqlssh=root
 SSH_PASSWORD_mysqlssh=xxx
 
 LABEL_sqlite=sqlite
-FILE_sqlite=/home/jena/dbgate-data/files/sqlite/feeds.sqlite
+FILE_sqlite=/home/jena/.dbgate/files/sqlite/feeds.sqlite
 ENGINE_sqlite=sqlite@dbgate-plugin-sqlite
 
+LABEL_relational=Relational dataset repo
+SERVER_relational=relational.fit.cvut.cz
+USER_relational=guest
+PASSWORD_relational=relational
+ENGINE_relational=mariadb@dbgate-plugin-mysql
+READONLY_relational=1
+
+# SETTINGS_dataGrid.showHintColumns=1
+
 # docker run -p 3000:3000 -e CONNECTIONS=mongo -e URL_mongo=mongodb://localhost:27017 -e ENGINE_mongo=mongo@dbgate-plugin-mongo -e LABEL_mongo=mongo dbgate/dbgate:beta
+
+# LOGINS=x,y
+# LOGIN_PASSWORD_x=x
+# LOGIN_PASSWORD_y=LOGIN_PASSWORD_y
+# LOGIN_PERMISSIONS_x=~*
+# LOGIN_PERMISSIONS_y=~*
+
+# PERMISSIONS=~*,connections/relational
+# PERMISSIONS=~*

packages/api/env/singledb/.env

@@ -5,9 +5,11 @@ CONNECTIONS=mysql
 LABEL_mysql=MySql localhost
 SERVER_mysql=localhost
 USER_mysql=root
-PASSWORD_mysql=test
-PORT_mysql=3307
+PASSWORD_mysql=Pwd2020Db
+PORT_mysql=3306
 ENGINE_mysql=mysql@dbgate-plugin-mysql
+DBCONFIG_mysql=[{"name":"Chinook","connectionColor":"cyan"}]
+
 
 SINGLE_CONNECTION=mysql
 SINGLE_DATABASE=Chinook

packages/api/package.json

@@ -1,7 +1,7 @@
 {
   "name": "dbgate-api",
   "main": "src/index.js",
-  "version": "4.1.1",
+  "version": "5.0.0-alpha.1",
   "homepage": "https://dbgate.org/",
   "repository": {
     "type": "git",
@@ -17,6 +17,7 @@
     "dbgate"
   ],
   "dependencies": {
+    "activedirectory2": "^2.1.0",
     "async-lock": "^1.2.4",
     "axios": "^0.21.1",
     "body-parser": "^1.19.0",
@@ -25,15 +26,18 @@
     "compare-versions": "^3.6.0",
     "cors": "^2.8.5",
     "cross-env": "^6.0.3",
-    "dbgate-query-splitter": "^4.1.1",
-    "dbgate-sqltree": "^4.1.1",
-    "dbgate-tools": "^4.1.1",
+    "dbgate-query-splitter": "^4.9.3",
+    "dbgate-sqltree": "^5.0.0-alpha.1",
+    "dbgate-tools": "^5.0.0-alpha.1",
+    "dbgate-datalib": "^5.0.0-alpha.1",
+    "debug": "^4.3.4",
     "diff": "^5.0.0",
     "diff2html": "^3.4.13",
     "eslint": "^6.8.0",
     "express": "^4.17.1",
     "express-basic-auth": "^1.2.0",
     "express-fileupload": "^1.2.0",
+    "external-sorting": "^1.3.1",
     "fs-extra": "^9.1.0",
     "fs-reverse": "^0.0.3",
     "get-port": "^5.1.1",
@@ -41,29 +45,36 @@
     "is-electron": "^2.2.1",
     "js-yaml": "^4.1.0",
     "json-stable-stringify": "^1.0.1",
+    "jsonwebtoken": "^8.5.1",
     "line-reader": "^0.4.0",
     "lodash": "^4.17.21",
+    "moment": "^2.24.0",
     "ncp": "^2.0.0",
     "node-cron": "^2.0.3",
-    "node-ssh-forward": "^0.7.2",
+    "on-finished": "^2.4.1",
+    "pinomin": "^1.0.1",
     "portfinder": "^1.0.28",
+    "rimraf": "^3.0.0",
     "simple-encryptor": "^4.0.0",
+    "ssh2": "^1.11.0",
     "tar": "^6.0.5",
     "uuid": "^3.4.0"
   },
   "scripts": {
-    "start": "env-cmd node src/index.js",
-    "start:portal": "env-cmd -f .env-portal node src/index.js",
-    "start:singledb": "env-cmd -f .env-singledb node src/index.js",
-    "start:filedb": "env-cmd node src/index.js /home/jena/test/chinook/Chinook.db",
-    "start:singleconn": "env-cmd node src/index.js --server localhost --user root --port 3307 --engine mysql@dbgate-plugin-mysql --password test",
+    "start": "env-cmd node src/index.js --listen-api",
+    "start:portal": "env-cmd -f env/portal/.env node src/index.js --listen-api",
+    "start:singledb": "env-cmd -f env/singledb/.env node src/index.js --listen-api",
+    "start:auth": "env-cmd -f env/auth/.env node src/index.js --listen-api",
+    "start:dblogin": "env-cmd -f env/dblogin/.env node src/index.js --listen-api",
+    "start:filedb": "env-cmd node src/index.js /home/jena/test/chinook/Chinook.db --listen-api",
+    "start:singleconn": "env-cmd node src/index.js --server localhost --user root --port 3307 --engine mysql@dbgate-plugin-mysql --password test --listen-api",
     "ts": "tsc",
     "build": "webpack"
   },
   "devDependencies": {
     "@types/fs-extra": "^9.0.11",
     "@types/lodash": "^4.14.149",
-    "dbgate-types": "^4.1.1",
+    "dbgate-types": "^5.0.0-alpha.1",
     "env-cmd": "^10.1.0",
     "node-loader": "^1.0.2",
     "nodemon": "^2.0.2",
@@ -72,7 +83,8 @@
     "webpack-cli": "^3.3.11"
   },
   "optionalDependencies": {
-    "better-sqlite3": "7.4.5",
-    "msnodesqlv8": "^2.4.4"
+    "better-sqlite3": "7.6.2",
+    "msnodesqlv8": "^2.6.0",
+    "oracledb": "^5.5.0"
   }
 }

packages/api/src/controllers/apps.js

@@ -58,7 +58,7 @@ module.exports = {
 
   refreshFiles_meta: true,
   async refreshFiles({ folder }) {
-    socket.emitChanged(`app-files-changed-${folder}`);
+    socket.emitChanged('app-files-changed', { app: folder });
   },
 
   refreshFolders_meta: true,
@@ -69,7 +69,7 @@ module.exports = {
   deleteFile_meta: true,
   async deleteFile({ folder, file, fileType }) {
     await fs.unlink(path.join(appdir(), folder, `${file}.${fileType}`));
-    socket.emitChanged(`app-files-changed-${folder}`);
+    socket.emitChanged('app-files-changed', { app: folder });
     this.emitChangedDbApp(folder);
   },
 
@@ -79,7 +79,7 @@ module.exports = {
       path.join(path.join(appdir(), folder), `${file}.${fileType}`),
       path.join(path.join(appdir(), folder), `${newFile}.${fileType}`)
     );
-    socket.emitChanged(`app-files-changed-${folder}`);
+    socket.emitChanged('app-files-changed', { app: folder });
     this.emitChangedDbApp(folder);
   },
 
@@ -95,7 +95,7 @@ module.exports = {
     if (!folder) throw new Error('Missing folder parameter');
     await fs.rmdir(path.join(appdir(), folder), { recursive: true });
     socket.emitChanged(`app-folders-changed`);
-    socket.emitChanged(`app-files-changed-${folder}`);
+    socket.emitChanged('app-files-changed', { app: folder });
     socket.emitChanged('used-apps-changed');
   },
 
@@ -219,7 +219,7 @@ module.exports = {
 
     await fs.writeFile(file, JSON.stringify(json, undefined, 2));
 
-    socket.emitChanged(`app-files-changed-${appFolder}`);
+    socket.emitChanged('app-files-changed', { app: appFolder });
     socket.emitChanged('used-apps-changed');
   },
 
@@ -265,4 +265,16 @@ module.exports = {
 
     return true;
   },
+
+  createConfigFile_meta: true,
+  async createConfigFile({ appFolder, fileName, content }) {
+    const file = path.join(appdir(), appFolder, fileName);
+    if (!(await fs.exists(file))) {
+      await fs.writeFile(file, JSON.stringify(content, undefined, 2));
+      socket.emitChanged('app-files-changed', { app: appFolder });
+      socket.emitChanged('used-apps-changed');
+      return true;
+    }
+    return false;
+  },
 };

packages/api/src/controllers/archive.js

@@ -1,13 +1,17 @@
 const fs = require('fs-extra');
-const stream = require('stream');
 const readline = require('readline');
 const path = require('path');
-const { formatWithOptions } = require('util');
 const { archivedir, clearArchiveLinksCache, resolveArchiveFolder } = require('../utility/directories');
 const socket = require('../utility/socket');
-const JsonLinesDatastore = require('../utility/JsonLinesDatastore');
-const { saveFreeTableData } = require('../utility/freeTableStorage');
 const loadFilesRecursive = require('../utility/loadFilesRecursive');
+const getJslFileName = require('../utility/getJslFileName');
+const { getLogger } = require('dbgate-tools');
+const uuidv1 = require('uuid/v1');
+const dbgateApi = require('../shell');
+const jsldata = require('./jsldata');
+const platformInfo = require('../utility/platformInfo');
+
+const logger = getLogger('archive');
 
 module.exports = {
   folders_meta: true,
@@ -45,45 +49,53 @@ module.exports = {
 
   files_meta: true,
   async files({ folder }) {
-    const dir = resolveArchiveFolder(folder);
-    if (!(await fs.exists(dir))) return [];
-    const files = await loadFilesRecursive(dir); // fs.readdir(dir);
+    try {
+      const dir = resolveArchiveFolder(folder);
+      if (!(await fs.exists(dir))) return [];
+      const files = await loadFilesRecursive(dir); // fs.readdir(dir);
 
-    function fileType(ext, type) {
-      return files
-        .filter(name => name.endsWith(ext))
-        .map(name => ({
-          name: name.slice(0, -ext.length),
-          label: path.parse(name.slice(0, -ext.length)).base,
-          type,
-        }));
+      function fileType(ext, type) {
+        return files
+          .filter(name => name.endsWith(ext))
+          .map(name => ({
+            name: name.slice(0, -ext.length),
+            label: path.parse(name.slice(0, -ext.length)).base,
+            type,
+          }));
+      }
+
+      return [
+        ...fileType('.jsonl', 'jsonl'),
+        ...fileType('.table.yaml', 'table.yaml'),
+        ...fileType('.view.sql', 'view.sql'),
+        ...fileType('.proc.sql', 'proc.sql'),
+        ...fileType('.func.sql', 'func.sql'),
+        ...fileType('.trigger.sql', 'trigger.sql'),
+        ...fileType('.matview.sql', 'matview.sql'),
+      ];
+    } catch (err) {
+      logger.error({ err }, 'Error reading archive files');
+      return [];
     }
-
-    return [
-      ...fileType('.jsonl', 'jsonl'),
-      ...fileType('.table.yaml', 'table.yaml'),
-      ...fileType('.view.sql', 'view.sql'),
-      ...fileType('.proc.sql', 'proc.sql'),
-      ...fileType('.func.sql', 'func.sql'),
-      ...fileType('.trigger.sql', 'trigger.sql'),
-      ...fileType('.matview.sql', 'matview.sql'),
-    ];
   },
 
   refreshFiles_meta: true,
   async refreshFiles({ folder }) {
-    socket.emitChanged(`archive-files-changed-${folder}`);
+    socket.emitChanged('archive-files-changed', { folder });
+    return true;
   },
 
   refreshFolders_meta: true,
   async refreshFolders() {
     socket.emitChanged(`archive-folders-changed`);
+    return true;
   },
 
   deleteFile_meta: true,
   async deleteFile({ folder, file, fileType }) {
     await fs.unlink(path.join(resolveArchiveFolder(folder), `${file}.${fileType}`));
-    socket.emitChanged(`archive-files-changed-${folder}`);
+    socket.emitChanged(`archive-files-changed`, { folder });
+    return true;
   },
 
   renameFile_meta: true,
@@ -92,7 +104,47 @@ module.exports = {
       path.join(resolveArchiveFolder(folder), `${file}.${fileType}`),
       path.join(resolveArchiveFolder(folder), `${newFile}.${fileType}`)
     );
-    socket.emitChanged(`archive-files-changed-${folder}`);
+    socket.emitChanged(`archive-files-changed`, { folder });
+    return true;
+  },
+
+  modifyFile_meta: true,
+  async modifyFile({ folder, file, changeSet, mergedRows, mergeKey, mergeMode }) {
+    await jsldata.closeDataStore(`archive://${folder}/${file}`);
+    const changedFilePath = path.join(resolveArchiveFolder(folder), `${file}.jsonl`);
+
+    if (!fs.existsSync(changedFilePath)) {
+      if (!mergedRows) {
+        return false;
+      }
+      const fileStream = fs.createWriteStream(changedFilePath);
+      for (const row of mergedRows) {
+        await fileStream.write(JSON.stringify(row) + '\n');
+      }
+      await fileStream.close();
+
+      socket.emitChanged(`archive-files-changed`, { folder });
+      return true;
+    }
+
+    const tmpchangedFilePath = path.join(resolveArchiveFolder(folder), `${file}-${uuidv1()}.jsonl`);
+    const reader = await dbgateApi.modifyJsonLinesReader({
+      fileName: changedFilePath,
+      changeSet,
+      mergedRows,
+      mergeKey,
+      mergeMode,
+    });
+    const writer = await dbgateApi.jsonLinesWriter({ fileName: tmpchangedFilePath });
+    await dbgateApi.copyStream(reader, writer);
+    if (platformInfo.isWindows) {
+      await fs.copyFile(tmpchangedFilePath, changedFilePath);
+      await fs.unlink(tmpchangedFilePath);
+    } else {
+      await fs.unlink(changedFilePath);
+      await fs.rename(tmpchangedFilePath, changedFilePath);
+    }
+    return true;
   },
 
   renameFolder_meta: true,
@@ -100,6 +152,7 @@ module.exports = {
     const uniqueName = await this.getNewArchiveFolder({ database: newFolder });
     await fs.rename(path.join(archivedir(), folder), path.join(archivedir(), uniqueName));
     socket.emitChanged(`archive-folders-changed`);
+    return true;
   },
 
   deleteFolder_meta: true,
@@ -111,40 +164,42 @@ module.exports = {
       await fs.rmdir(path.join(archivedir(), folder), { recursive: true });
     }
     socket.emitChanged(`archive-folders-changed`);
-  },
-
-  saveFreeTable_meta: true,
-  async saveFreeTable({ folder, file, data }) {
-    await saveFreeTableData(path.join(resolveArchiveFolder(folder), `${file}.jsonl`), data);
-    socket.emitChanged(`archive-files-changed-${folder}`);
     return true;
   },
 
-  loadFreeTable_meta: true,
-  async loadFreeTable({ folder, file }) {
-    return new Promise((resolve, reject) => {
-      const fileStream = fs.createReadStream(path.join(resolveArchiveFolder(folder), `${file}.jsonl`));
-      const liner = readline.createInterface({
-        input: fileStream,
-      });
-      let structure = null;
-      const rows = [];
-      liner.on('line', line => {
-        const data = JSON.parse(line);
-        if (structure) rows.push(data);
-        else structure = data;
-      });
-      liner.on('close', () => {
-        resolve({ structure, rows });
-        fileStream.close();
-      });
-    });
-  },
-
   saveText_meta: true,
   async saveText({ folder, file, text }) {
     await fs.writeFile(path.join(resolveArchiveFolder(folder), `${file}.jsonl`), text);
-    socket.emitChanged(`archive-files-changed-${folder}`);
+    socket.emitChanged(`archive-files-changed`, { folder });
+    return true;
+  },
+
+  saveJslData_meta: true,
+  async saveJslData({ folder, file, jslid, changeSet }) {
+    const source = getJslFileName(jslid);
+    const target = path.join(resolveArchiveFolder(folder), `${file}.jsonl`);
+    if (changeSet) {
+      const reader = await dbgateApi.modifyJsonLinesReader({
+        fileName: source,
+        changeSet,
+      });
+      const writer = await dbgateApi.jsonLinesWriter({ fileName: target });
+      await dbgateApi.copyStream(reader, writer);
+    } else {
+      await fs.copyFile(source, target);
+      socket.emitChanged(`archive-files-changed`, { folder });
+    }
+    return true;
+  },
+
+  saveRows_meta: true,
+  async saveRows({ folder, file, rows }) {
+    const fileStream = fs.createWriteStream(path.join(resolveArchiveFolder(folder), `${file}.jsonl`));
+    for (const row of rows) {
+      await fileStream.write(JSON.stringify(row) + '\n');
+    }
+    await fileStream.close();
+    socket.emitChanged(`archive-files-changed`, { folder });
     return true;
   },
 

packages/api/src/controllers/auth.js

@@ -0,0 +1,150 @@
+const axios = require('axios');
+const jwt = require('jsonwebtoken');
+const getExpressPath = require('../utility/getExpressPath');
+const uuidv1 = require('uuid/v1');
+const { getLogins } = require('../utility/hasPermission');
+const { getLogger } = require('dbgate-tools');
+const AD = require('activedirectory2').promiseWrapper;
+
+const logger = getLogger('auth');
+
+const tokenSecret = uuidv1();
+
+function shouldAuthorizeApi() {
+  const logins = getLogins();
+  return !!process.env.OAUTH_AUTH || !!process.env.AD_URL || (!!logins && !process.env.BASIC_AUTH);
+}
+
+function getTokenLifetime() {
+  return process.env.TOKEN_LIFETIME || '1d';
+}
+
+function unauthorizedResponse(req, res, text) {
+  // if (req.path == getExpressPath('/config/get-settings')) {
+  //   return res.json({});
+  // }
+  // if (req.path == getExpressPath('/connections/list')) {
+  //   return res.json([]);
+  // }
+  return res.sendStatus(401).send(text);
+}
+
+function authMiddleware(req, res, next) {
+  const SKIP_AUTH_PATHS = ['/config/get', '/auth/oauth-token', '/auth/login', '/stream'];
+
+  if (!shouldAuthorizeApi()) {
+    return next();
+  }
+  let skipAuth = !!SKIP_AUTH_PATHS.find(x => req.path == getExpressPath(x));
+
+  const authHeader = req.headers.authorization;
+  if (!authHeader) {
+    if (skipAuth) {
+      return next();
+    }
+    return unauthorizedResponse(req, res, 'missing authorization header');
+  }
+  const token = authHeader.split(' ')[1];
+  try {
+    const decoded = jwt.verify(token, tokenSecret);
+    req.user = decoded;
+    return next();
+  } catch (err) {
+    if (skipAuth) {
+      return next();
+    }
+
+    logger.error({ err }, 'Sending invalid token error');
+
+    return unauthorizedResponse(req, res, 'invalid token');
+  }
+}
+
+module.exports = {
+  oauthToken_meta: true,
+  async oauthToken(params) {
+    const { redirectUri, code } = params;
+
+    const scopeParam = process.env.OAUTH_SCOPE ? `&scope=${process.env.OAUTH_SCOPE}` : '';
+    const resp = await axios.default.post(
+      `${process.env.OAUTH_TOKEN}`,
+      `grant_type=authorization_code&code=${encodeURIComponent(code)}&redirect_uri=${encodeURIComponent(
+        redirectUri
+      )}&client_id=${process.env.OAUTH_CLIENT_ID}&client_secret=${process.env.OAUTH_CLIENT_SECRET}${scopeParam}`
+    );
+
+    const { access_token, refresh_token } = resp.data;
+
+    const payload = jwt.decode(access_token);
+
+    logger.info({ payload }, 'User payload returned from OAUTH');
+
+    const login =
+      process.env.OAUTH_LOGIN_FIELD && payload && payload[process.env.OAUTH_LOGIN_FIELD]
+        ? payload[process.env.OAUTH_LOGIN_FIELD]
+        : 'oauth';
+
+    if (
+      process.env.OAUTH_ALLOWED_LOGINS &&
+      !process.env.OAUTH_ALLOWED_LOGINS.split(',').find(x => x.toLowerCase().trim() == login.toLowerCase().trim())
+    ) {
+      return { error: `Username ${login} not allowed to log in` };
+    }
+    if (access_token) {
+      return {
+        accessToken: jwt.sign({ login }, tokenSecret, { expiresIn: getTokenLifetime() }),
+      };
+    }
+
+    return { error: 'Token not found' };
+  },
+  login_meta: true,
+  async login(params) {
+    const { login, password } = params;
+
+    if (process.env.AD_URL) {
+      const adConfig = {
+        url: process.env.AD_URL,
+        baseDN: process.env.AD_BASEDN,
+        username: process.env.AD_USERNAME,
+        password: process.env.AD_PASSOWRD,
+      };
+      const ad = new AD(adConfig);
+      try {
+        const res = await ad.authenticate(login, password);
+        if (!res) {
+          return { error: 'Login failed' };
+        }
+        if (
+          process.env.AD_ALLOWED_LOGINS &&
+          !process.env.AD_ALLOWED_LOGINS.split(',').find(x => x.toLowerCase().trim() == login.toLowerCase().trim())
+        ) {
+          return { error: `Username ${login} not allowed to log in` };
+        }
+        return {
+          accessToken: jwt.sign({ login }, tokenSecret, { expiresIn: getTokenLifetime() }),
+        };
+      } catch (err) {
+        logger.error({ err }, 'Failed active directory authentization');
+        return {
+          error: err.message,
+        };
+      }
+    }
+
+    const logins = getLogins();
+    if (!logins) {
+      return { error: 'Logins not configured' };
+    }
+    const foundLogin = logins.find(x => x.login == login);
+    if (foundLogin && foundLogin.password == password) {
+      return {
+        accessToken: jwt.sign({ login }, tokenSecret, { expiresIn: getTokenLifetime() }),
+      };
+    }
+    return { error: 'Invalid credentials' };
+  },
+
+  authMiddleware,
+  shouldAuthorizeApi,
+};

packages/api/src/controllers/config.js

@@ -1,38 +1,67 @@
 const fs = require('fs-extra');
+const os = require('os');
 const path = require('path');
 const axios = require('axios');
-const { datadir } = require('../utility/directories');
-const hasPermission = require('../utility/hasPermission');
+const { datadir, getLogsFilePath } = require('../utility/directories');
+const { hasPermission, getLogins } = require('../utility/hasPermission');
 const socket = require('../utility/socket');
 const _ = require('lodash');
+const AsyncLock = require('async-lock');
 
 const currentVersion = require('../currentVersion');
 const platformInfo = require('../utility/platformInfo');
 const connections = require('../controllers/connections');
 
-module.exports = {
-  settingsValue: {},
+const lock = new AsyncLock();
 
-  async _init() {
-    try {
-      this.settingsValue = JSON.parse(await fs.readFile(path.join(datadir(), 'settings.json'), { encoding: 'utf-8' }));
-    } catch (err) {
-      this.settingsValue = {};
-    }
-  },
+module.exports = {
+  // settingsValue: {},
+
+  // async _init() {
+  //   try {
+  //     this.settingsValue = JSON.parse(await fs.readFile(path.join(datadir(), 'settings.json'), { encoding: 'utf-8' }));
+  //   } catch (err) {
+  //     this.settingsValue = {};
+  //   }
+  // },
 
   get_meta: true,
-  async get() {
-    const permissions = process.env.PERMISSIONS ? process.env.PERMISSIONS.split(',') : null;
+  async get(_params, req) {
+    const logins = getLogins();
+    const loginName =
+      req && req.user && req.user.login ? req.user.login : req && req.auth && req.auth.user ? req.auth.user : null;
+    const login = logins && loginName ? logins.find(x => x.login == loginName) : null;
+    const permissions = login ? login.permissions : process.env.PERMISSIONS;
 
     return {
       runAsPortal: !!connections.portalConnections,
-      singleDatabase: connections.singleDatabase,
+      singleDbConnection: connections.singleDbConnection,
+      singleConnection: connections.singleConnection,
+      // hideAppEditor: !!process.env.HIDE_APP_EDITOR,
+      allowShellConnection: platformInfo.allowShellConnection,
+      allowShellScripting: platformInfo.allowShellScripting,
+      isDocker: platformInfo.isDocker,
       permissions,
+      login,
+      oauth: process.env.OAUTH_AUTH,
+      oauthClient: process.env.OAUTH_CLIENT_ID,
+      oauthScope: process.env.OAUTH_SCOPE,
+      oauthLogout: process.env.OAUTH_LOGOUT,
+      isLoginForm: !!process.env.AD_URL || (!!logins && !process.env.BASIC_AUTH),
+      logsFilePath: getLogsFilePath(),
+      connectionsFilePath: path.join(datadir(), 'connections.jsonl'),
       ...currentVersion,
     };
   },
 
+  logout_meta: {
+    method: 'get',
+    raw: true,
+  },
+  logout(req, res) {
+    res.status(401).send('Logged out<br><a href="../..">Back to DbGate</a>');
+  },
+
   platformInfo_meta: true,
   async platformInfo() {
     return platformInfo;
@@ -40,24 +69,60 @@ module.exports = {
 
   getSettings_meta: true,
   async getSettings() {
-    return this.settingsValue;
+    const res = await lock.acquire('settings', async () => {
+      return await this.loadSettings();
+    });
+    return res;
+  },
+
+  fillMissingSettings(value) {
+    const res = {
+      ...value,
+    };
+    if (value['app.useNativeMenu'] !== true && value['app.useNativeMenu'] !== false) {
+      // res['app.useNativeMenu'] = os.platform() == 'darwin' ? true : false;
+      res['app.useNativeMenu'] = false;
+    }
+    for (const envVar in process.env) {
+      if (envVar.startsWith('SETTINGS_')) {
+        const key = envVar.substring('SETTINGS_'.length);
+        if (!res[key]) {
+          res[key] = process.env[envVar];
+        }
+      }
+    }
+    return res;
+  },
+
+  async loadSettings() {
+    try {
+      const settingsText = await fs.readFile(path.join(datadir(), 'settings.json'), { encoding: 'utf-8' });
+      return this.fillMissingSettings(JSON.parse(settingsText));
+    } catch (err) {
+      return this.fillMissingSettings({});
+    }
   },
 
   updateSettings_meta: true,
-  async updateSettings(values) {
-    if (!hasPermission(`settings/change`)) return false;
-    try {
-      const updated = {
-        ...this.settingsValue,
-        ...values,
-      };
-      await fs.writeFile(path.join(datadir(), 'settings.json'), JSON.stringify(updated, undefined, 2));
-      this.settingsValue = updated;
-      socket.emitChanged(`settings-changed`);
-      return updated;
-    } catch (err) {
-      return false;
-    }
+  async updateSettings(values, req) {
+    if (!hasPermission(`settings/change`, req)) return false;
+
+    const res = await lock.acquire('settings', async () => {
+      const currentValue = await this.loadSettings();
+      try {
+        const updated = {
+          ...currentValue,
+          ...values,
+        };
+        await fs.writeFile(path.join(datadir(), 'settings.json'), JSON.stringify(updated, undefined, 2));
+        // this.settingsValue = updated;
+        socket.emitChanged(`settings-changed`);
+        return updated;
+      } catch (err) {
+        return false;
+      }
+    });
+    return res;
   },
 
   changelog_meta: true,

packages/api/src/controllers/connections.js

@@ -2,15 +2,24 @@ const path = require('path');
 const { fork } = require('child_process');
 const _ = require('lodash');
 const fs = require('fs-extra');
+const crypto = require('crypto');
 
 const { datadir, filesdir } = require('../utility/directories');
 const socket = require('../utility/socket');
-const { encryptConnection } = require('../utility/crypting');
+const { encryptConnection, maskConnection } = require('../utility/crypting');
 const { handleProcessCommunication } = require('../utility/processComm');
 const { pickSafeConnectionInfo } = require('../utility/crypting');
 const JsonLinesDatabase = require('../utility/JsonLinesDatabase');
 
 const processArgs = require('../utility/processArgs');
+const { safeJsonParse, getLogger } = require('dbgate-tools');
+const platformInfo = require('../utility/platformInfo');
+const { connectionHasPermission, testConnectionPermission } = require('../utility/hasPermission');
+const pipeForkLogs = require('../utility/pipeForkLogs');
+
+const logger = getLogger('connections');
+
+let volatileConnections = {};
 
 function getNamedArgs() {
   const res = {};
@@ -46,15 +55,21 @@ function getPortalCollections() {
       server: process.env[`SERVER_${id}`],
       user: process.env[`USER_${id}`],
       password: process.env[`PASSWORD_${id}`],
+      passwordMode: process.env[`PASSWORD_MODE_${id}`],
       port: process.env[`PORT_${id}`],
       databaseUrl: process.env[`URL_${id}`],
       useDatabaseUrl: !!process.env[`URL_${id}`],
       databaseFile: process.env[`FILE_${id}`],
+      socketPath: process.env[`SOCKET_PATH_${id}`],
+      authType: process.env[`AUTH_TYPE_${id}`] || (process.env[`SOCKET_PATH_${id}`] ? 'socket' : undefined),
       defaultDatabase:
         process.env[`DATABASE_${id}`] ||
         (process.env[`FILE_${id}`] ? getDatabaseFileLabel(process.env[`FILE_${id}`]) : null),
       singleDatabase: !!process.env[`DATABASE_${id}`] || !!process.env[`FILE_${id}`],
       displayName: process.env[`LABEL_${id}`],
+      isReadOnly: process.env[`READONLY_${id}`],
+      databases: process.env[`DBCONFIG_${id}`] ? safeJsonParse(process.env[`DBCONFIG_${id}`]) : null,
+      parent: process.env[`PARENT_${id}`] || undefined,
 
       // SSH tunnel
       useSshTunnel: process.env[`USE_SSH_${id}`],
@@ -74,13 +89,13 @@ function getPortalCollections() {
       sslKeyFile: process.env[`SSL_KEY_FILE_${id}`],
       sslRejectUnauthorized: process.env[`SSL_REJECT_UNAUTHORIZED_${id}`],
     }));
-    console.log('Using connections from ENV variables:');
-    console.log(JSON.stringify(connections.map(pickSafeConnectionInfo), undefined, 2));
+
+    logger.info({ connections: connections.map(pickSafeConnectionInfo) }, 'Using connections from ENV variables');
     const noengine = connections.filter(x => !x.engine);
     if (noengine.length > 0) {
-      console.log(
-        'Warning: Invalid CONNECTIONS configutation, missing ENGINE for connection ID:',
-        noengine.map(x => x._id)
+      logger.warn(
+        { connections: noengine.map(x => x._id) },
+        'Invalid CONNECTIONS configutation, missing ENGINE for connection ID'
       );
     }
     return connections;
@@ -118,9 +133,10 @@ function getPortalCollections() {
 
   return null;
 }
+
 const portalConnections = getPortalCollections();
 
-function getSingleDatabase() {
+function getSingleDbConnection() {
   if (process.env.SINGLE_CONNECTION && process.env.SINGLE_DATABASE) {
     // @ts-ignore
     const connection = portalConnections.find(x => x._id == process.env.SINGLE_CONNECTION);
@@ -144,12 +160,31 @@ function getSingleDatabase() {
   return null;
 }
 
-const singleDatabase = getSingleDatabase();
+function getSingleConnection() {
+  if (getSingleDbConnection()) return null;
+  if (process.env.SINGLE_CONNECTION) {
+    // @ts-ignore
+    const connection = portalConnections.find(x => x._id == process.env.SINGLE_CONNECTION);
+    if (connection) {
+      return connection;
+    }
+  }
+  // @ts-ignore
+  const arg0 = (portalConnections || []).find(x => x._id == 'argv');
+  if (arg0) {
+    return arg0;
+  }
+  return null;
+}
+
+const singleDbConnection = getSingleDbConnection();
+const singleConnection = getSingleConnection();
 
 module.exports = {
   datastore: null,
   opened: [],
-  singleDatabase,
+  singleDbConnection,
+  singleConnection,
   portalConnections,
 
   async _init() {
@@ -161,19 +196,30 @@ module.exports = {
   },
 
   list_meta: true,
-  async list() {
-    return portalConnections || this.datastore.find();
+  async list(_params, req) {
+    if (portalConnections) {
+      if (platformInfo.allowShellConnection) return portalConnections;
+      return portalConnections.map(maskConnection).filter(x => connectionHasPermission(x, req));
+    }
+    return (await this.datastore.find()).filter(x => connectionHasPermission(x, req));
   },
 
   test_meta: true,
   test(connection) {
-    const subprocess = fork(global['API_PACKAGE'] || process.argv[1], [
-      '--is-forked-api',
-      '--start-process',
-      'connectProcess',
-      ...processArgs.getPassArgs(),
-      // ...process.argv.slice(3),
-    ]);
+    const subprocess = fork(
+      global['API_PACKAGE'] || process.argv[1],
+      [
+        '--is-forked-api',
+        '--start-process',
+        'connectProcess',
+        ...processArgs.getPassArgs(),
+        // ...process.argv.slice(3),
+      ],
+      {
+        stdio: ['ignore', 'pipe', 'pipe', 'ipc'],
+      }
+    );
+    pipeForkLogs(subprocess);
     subprocess.send(connection);
     return new Promise(resolve => {
       subprocess.on('message', resp => {
@@ -187,6 +233,36 @@ module.exports = {
     });
   },
 
+  saveVolatile_meta: true,
+  async saveVolatile({ conid, user, password, test }) {
+    const old = await this.getCore({ conid });
+    const res = {
+      ...old,
+      _id: crypto.randomUUID(),
+      password,
+      passwordMode: undefined,
+      unsaved: true,
+    };
+    if (old.passwordMode == 'askUser') {
+      res.user = user;
+    }
+
+    if (test) {
+      const testRes = await this.test(res);
+      if (testRes.msgtype == 'connected') {
+        volatileConnections[res._id] = res;
+        return {
+          ...res,
+          msgtype: 'connected',
+        };
+      }
+      return testRes;
+    } else {
+      volatileConnections[res._id] = res;
+      return res;
+    }
+  },
+
   save_meta: true,
   async save(connection) {
     if (portalConnections) return;
@@ -199,6 +275,9 @@ module.exports = {
     }
     socket.emitChanged('connection-list-changed');
     socket.emitChanged('used-apps-changed');
+    if (this._closeAll) {
+      this._closeAll(connection._id);
+    }
     // for (const db of connection.databases || []) {
     //   socket.emitChanged(`db-apps-changed-${connection._id}-${db.name}`);
     // }
@@ -206,16 +285,26 @@ module.exports = {
   },
 
   update_meta: true,
-  async update({ _id, values }) {
+  async update({ _id, values }, req) {
     if (portalConnections) return;
+    testConnectionPermission(_id, req);
     const res = await this.datastore.patch(_id, values);
     socket.emitChanged('connection-list-changed');
     return res;
   },
 
+  batchChangeFolder_meta: true,
+  async batchChangeFolder({ folder, newFolder }, req) {
+    // const updated = await this.datastore.find(x => x.parent == folder);
+    const res = await this.datastore.updateAll(x => (x.parent == folder ? { ...x, parent: newFolder } : x));
+    socket.emitChanged('connection-list-changed');
+    return res;
+  },
+
   updateDatabase_meta: true,
-  async updateDatabase({ conid, database, values }) {
+  async updateDatabase({ conid, database, values }, req) {
     if (portalConnections) return;
+    testConnectionPermission(conid, req);
     const conn = await this.datastore.get(conid);
     let databases = (conn && conn.databases) || [];
     if (databases.find(x => x.name == database)) {
@@ -231,20 +320,34 @@ module.exports = {
   },
 
   delete_meta: true,
-  async delete(connection) {
+  async delete(connection, req) {
     if (portalConnections) return;
+    testConnectionPermission(connection, req);
     const res = await this.datastore.remove(connection._id);
     socket.emitChanged('connection-list-changed');
     return res;
   },
 
-  get_meta: true,
-  async get({ conid }) {
-    if (portalConnections) return portalConnections.find(x => x._id == conid) || null;
+  async getCore({ conid, mask = false }) {
+    if (!conid) return null;
+    const volatile = volatileConnections[conid];
+    if (volatile) {
+      return volatile;
+    }
+    if (portalConnections) {
+      const res = portalConnections.find(x => x._id == conid) || null;
+      return mask && !platformInfo.allowShellConnection ? maskConnection(res) : res;
+    }
     const res = await this.datastore.get(conid);
     return res || null;
   },
 
+  get_meta: true,
+  async get({ conid }, req) {
+    testConnectionPermission(conid, req);
+    return this.getCore({ conid, mask: true });
+  },
+
   newSqliteDatabase_meta: true,
   async newSqliteDatabase({ file }) {
     const sqliteDir = path.join(filesdir(), 'sqlite');

packages/api/src/controllers/databaseConnections.js

@@ -12,6 +12,7 @@ const {
   matchPairedObjects,
   extendDatabaseInfo,
   modelCompareDbDiffOptions,
+  getLogger,
 } = require('dbgate-tools');
 const { html, parse } = require('diff2html');
 const { handleProcessCommunication } = require('../utility/processComm');
@@ -26,6 +27,11 @@ const generateDeploySql = require('../shell/generateDeploySql');
 const { createTwoFilesPatch } = require('diff');
 const diff2htmlPage = require('../utility/diff2htmlPage');
 const processArgs = require('../utility/processArgs');
+const { testConnectionPermission } = require('../utility/hasPermission');
+const { MissingCredentialsError } = require('../utility/exceptions');
+const pipeForkLogs = require('../utility/pipeForkLogs');
+
+const logger = getLogger('databaseConnections');
 
 module.exports = {
   /** @type {import('dbgate-types').OpenedDatabaseConnection[]} */
@@ -33,28 +39,32 @@ module.exports = {
   closed: {},
   requests: {},
 
+  async _init() {
+    connections._closeAll = conid => this.closeAll(conid);
+  },
+
   handle_structure(conid, database, { structure }) {
     const existing = this.opened.find(x => x.conid == conid && x.database == database);
     if (!existing) return;
     existing.structure = structure;
-    socket.emitChanged(`database-structure-changed-${conid}-${database}`);
+    socket.emitChanged('database-structure-changed', { conid, database });
   },
   handle_structureTime(conid, database, { analysedTime }) {
     const existing = this.opened.find(x => x.conid == conid && x.database == database);
     if (!existing) return;
     existing.analysedTime = analysedTime;
-    socket.emitChanged(`database-status-changed-${conid}-${database}`);
+    socket.emitChanged(`database-status-changed`, { conid, database });
   },
   handle_version(conid, database, { version }) {
     const existing = this.opened.find(x => x.conid == conid && x.database == database);
     if (!existing) return;
     existing.serverVersion = version;
-    socket.emitChanged(`database-server-version-changed-${conid}-${database}`);
+    socket.emitChanged(`database-server-version-changed`, { conid, database });
   },
 
   handle_error(conid, database, props) {
     const { error } = props;
-    console.log(`Error in database connection ${conid}, database ${database}: ${error}`);
+    logger.error(`Error in database connection ${conid}, database ${database}: ${error}`);
   },
   handle_response(conid, database, { msgid, ...response }) {
     const [resolve, reject] = this.requests[msgid];
@@ -67,7 +77,7 @@ module.exports = {
     if (!existing) return;
     if (existing.status && status && existing.status.counter > status.counter) return;
     existing.status = status;
-    socket.emitChanged(`database-status-changed-${conid}-${database}`);
+    socket.emitChanged(`database-status-changed`, { conid, database });
   },
 
   handle_ping() {},
@@ -75,14 +85,24 @@ module.exports = {
   async ensureOpened(conid, database) {
     const existing = this.opened.find(x => x.conid == conid && x.database == database);
     if (existing) return existing;
-    const connection = await connections.get({ conid });
-    const subprocess = fork(global['API_PACKAGE'] || process.argv[1], [
-      '--is-forked-api',
-      '--start-process',
-      'databaseConnectionProcess',
-      ...processArgs.getPassArgs(),
-      // ...process.argv.slice(3),
-    ]);
+    const connection = await connections.getCore({ conid });
+    if (connection.passwordMode == 'askPassword' || connection.passwordMode == 'askUser') {
+      throw new MissingCredentialsError({ conid, passwordMode: connection.passwordMode });
+    }
+    const subprocess = fork(
+      global['API_PACKAGE'] || process.argv[1],
+      [
+        '--is-forked-api',
+        '--start-process',
+        'databaseConnectionProcess',
+        ...processArgs.getPassArgs(),
+        // ...process.argv.slice(3),
+      ],
+      {
+        stdio: ['ignore', 'pipe', 'pipe', 'ipc'],
+      }
+    );
+    pipeForkLogs(subprocess);
     const lastClosed = this.closed[`${conid}/${database}`];
     const newOpened = {
       conid,
@@ -110,7 +130,7 @@ module.exports = {
       msgtype: 'connect',
       connection: { ...connection, database },
       structure: lastClosed ? lastClosed.structure : null,
-      globalSettings: config.settingsValue,
+      globalSettings: await config.getSettings(),
     });
     return newOpened;
   },
@@ -120,14 +140,20 @@ module.exports = {
     const msgid = uuidv1();
     const promise = new Promise((resolve, reject) => {
       this.requests[msgid] = [resolve, reject];
-      conn.subprocess.send({ msgid, ...message });
+      try {
+        conn.subprocess.send({ msgid, ...message });
+      } catch (err) {
+        logger.error({ err }, 'Error sending request do process');
+        this.close(conn.conid, conn.database);
+      }
     });
     return promise;
   },
 
   queryData_meta: true,
-  async queryData({ conid, database, sql }) {
-    console.log(`Processing query, conid=${conid}, database=${database}, sql=${sql}`);
+  async queryData({ conid, database, sql }, req) {
+    testConnectionPermission(conid, req);
+    logger.info({ conid, database, sql }, 'Processing query');
     const opened = await this.ensureOpened(conid, database);
     // if (opened && opened.status && opened.status.name == 'error') {
     //   return opened.status;
@@ -136,30 +162,110 @@ module.exports = {
     return res;
   },
 
-  runScript_meta: true,
-  async runScript({ conid, database, sql }) {
-    console.log(`Processing script, conid=${conid}, database=${database}, sql=${sql}`);
+  sqlSelect_meta: true,
+  async sqlSelect({ conid, database, select }, req) {
+    testConnectionPermission(conid, req);
     const opened = await this.ensureOpened(conid, database);
-    const res = await this.sendRequest(opened, { msgtype: 'runScript', sql });
+    const res = await this.sendRequest(opened, { msgtype: 'sqlSelect', select });
+    return res;
+  },
+
+  runScript_meta: true,
+  async runScript({ conid, database, sql, useTransaction }, req) {
+    testConnectionPermission(conid, req);
+    logger.info({ conid, database, sql }, 'Processing script');
+    const opened = await this.ensureOpened(conid, database);
+    const res = await this.sendRequest(opened, { msgtype: 'runScript', sql, useTransaction });
     return res;
   },
 
   collectionData_meta: true,
-  async collectionData({ conid, database, options }) {
+  async collectionData({ conid, database, options }, req) {
+    testConnectionPermission(conid, req);
     const opened = await this.ensureOpened(conid, database);
     const res = await this.sendRequest(opened, { msgtype: 'collectionData', options });
-    return res.result;
+    return res.result || null;
+  },
+
+  async loadDataCore(msgtype, { conid, database, ...args }, req) {
+    testConnectionPermission(conid, req);
+    const opened = await this.ensureOpened(conid, database);
+    const res = await this.sendRequest(opened, { msgtype, ...args });
+    if (res.errorMessage) {
+      console.error(res.errorMessage);
+
+      return {
+        errorMessage: res.errorMessage,
+      };
+    }
+    return res.result || null;
+  },
+
+  loadKeys_meta: true,
+  async loadKeys({ conid, database, root, filter }, req) {
+    testConnectionPermission(conid, req);
+    return this.loadDataCore('loadKeys', { conid, database, root, filter });
+  },
+
+  exportKeys_meta: true,
+  async exportKeys({ conid, database, options }, req) {
+    testConnectionPermission(conid, req);
+    return this.loadDataCore('exportKeys', { conid, database, options });
+  },
+
+  loadKeyInfo_meta: true,
+  async loadKeyInfo({ conid, database, key }, req) {
+    testConnectionPermission(conid, req);
+    return this.loadDataCore('loadKeyInfo', { conid, database, key });
+  },
+
+  loadKeyTableRange_meta: true,
+  async loadKeyTableRange({ conid, database, key, cursor, count }, req) {
+    testConnectionPermission(conid, req);
+    return this.loadDataCore('loadKeyTableRange', { conid, database, key, cursor, count });
+  },
+
+  loadFieldValues_meta: true,
+  async loadFieldValues({ conid, database, schemaName, pureName, field, search }, req) {
+    testConnectionPermission(conid, req);
+    return this.loadDataCore('loadFieldValues', { conid, database, schemaName, pureName, field, search });
+  },
+
+  callMethod_meta: true,
+  async callMethod({ conid, database, method, args }, req) {
+    testConnectionPermission(conid, req);
+    return this.loadDataCore('callMethod', { conid, database, method, args });
+
+    // const opened = await this.ensureOpened(conid, database);
+    // const res = await this.sendRequest(opened, { msgtype: 'callMethod', method, args });
+    // if (res.errorMessage) {
+    //   console.error(res.errorMessage);
+    // }
+    // return res.result || null;
   },
 
   updateCollection_meta: true,
-  async updateCollection({ conid, database, changeSet }) {
+  async updateCollection({ conid, database, changeSet }, req) {
+    testConnectionPermission(conid, req);
     const opened = await this.ensureOpened(conid, database);
     const res = await this.sendRequest(opened, { msgtype: 'updateCollection', changeSet });
-    return res.result;
+    if (res.errorMessage) {
+      return {
+        errorMessage: res.errorMessage,
+      };
+    }
+    return res.result || null;
   },
 
   status_meta: true,
-  async status({ conid, database }) {
+  async status({ conid, database }, req) {
+    if (!conid) {
+      return {
+        name: 'error',
+        message: 'No connection',
+      };
+    }
+    testConnectionPermission(conid, req);
     const existing = this.opened.find(x => x.conid == conid && x.database == database);
     if (existing) {
       return {
@@ -181,12 +287,14 @@ module.exports = {
   },
 
   ping_meta: true,
-  async ping({ conid, database }) {
+  async ping({ conid, database }, req) {
+    testConnectionPermission(conid, req);
     let existing = this.opened.find(x => x.conid == conid && x.database == database);
 
     if (existing) {
       existing.subprocess.send({ msgtype: 'ping' });
     } else {
+      // @ts-ignore
       existing = await this.ensureOpened(conid, database);
     }
 
@@ -197,7 +305,8 @@ module.exports = {
   },
 
   refresh_meta: true,
-  async refresh({ conid, database, keepOpen }) {
+  async refresh({ conid, database, keepOpen }, req) {
+    testConnectionPermission(conid, req);
     if (!keepOpen) this.close(conid, database);
 
     await this.ensureOpened(conid, database);
@@ -205,9 +314,10 @@ module.exports = {
   },
 
   syncModel_meta: true,
-  async syncModel({ conid, database }) {
+  async syncModel({ conid, database, isFullRefresh }, req) {
+    testConnectionPermission(conid, req);
     const conn = await this.ensureOpened(conid, database);
-    conn.subprocess.send({ msgtype: 'syncModel' });
+    conn.subprocess.send({ msgtype: 'syncModel', isFullRefresh });
     return { status: 'ok' };
   },
 
@@ -215,7 +325,13 @@ module.exports = {
     const existing = this.opened.find(x => x.conid == conid && x.database == database);
     if (existing) {
       existing.disconnected = true;
-      if (kill) existing.subprocess.kill();
+      if (kill) {
+        try {
+          existing.subprocess.kill();
+        } catch (err) {
+          logger.error({ err }, 'Error killing subprocess');
+        }
+      }
       this.opened = this.opened.filter(x => x.conid != conid || x.database != database);
       this.closed[`${conid}/${database}`] = {
         status: {
@@ -224,18 +340,26 @@ module.exports = {
         },
         structure: existing.structure,
       };
-      socket.emitChanged(`database-status-changed-${conid}-${database}`);
+      socket.emitChanged(`database-status-changed`, { conid, database });
+    }
+  },
+
+  closeAll(conid, kill = true) {
+    for (const existing of this.opened.filter(x => x.conid == conid)) {
+      this.close(conid, existing.database, kill);
     }
   },
 
   disconnect_meta: true,
-  async disconnect({ conid, database }) {
+  async disconnect({ conid, database }, req) {
+    testConnectionPermission(conid, req);
     await this.close(conid, database, true);
     return { status: 'ok' };
   },
 
   structure_meta: true,
-  async structure({ conid, database }) {
+  async structure({ conid, database }, req) {
+    testConnectionPermission(conid, req);
     if (conid == '__model') {
       const model = await importDbModel(database);
       return model;
@@ -252,14 +376,19 @@ module.exports = {
   },
 
   serverVersion_meta: true,
-  async serverVersion({ conid, database }) {
+  async serverVersion({ conid, database }, req) {
+    if (!conid) {
+      return null;
+    }
+    testConnectionPermission(conid, req);
     if (!conid) return null;
     const opened = await this.ensureOpened(conid, database);
     return opened.serverVersion || null;
   },
 
   sqlPreview_meta: true,
-  async sqlPreview({ conid, database, objects, options }) {
+  async sqlPreview({ conid, database, objects, options }, req) {
+    testConnectionPermission(conid, req);
     // wait for structure
     await this.structure({ conid, database });
 
@@ -269,7 +398,8 @@ module.exports = {
   },
 
   exportModel_meta: true,
-  async exportModel({ conid, database }) {
+  async exportModel({ conid, database }, req) {
+    testConnectionPermission(conid, req);
     const archiveFolder = await archive.getNewArchiveFolder({ database });
     await fs.mkdir(path.join(archivedir(), archiveFolder));
     const model = await this.structure({ conid, database });
@@ -279,7 +409,8 @@ module.exports = {
   },
 
   generateDeploySql_meta: true,
-  async generateDeploySql({ conid, database, archiveFolder }) {
+  async generateDeploySql({ conid, database, archiveFolder }, req) {
+    testConnectionPermission(conid, req);
     const opened = await this.ensureOpened(conid, database);
     const res = await this.sendRequest(opened, {
       msgtype: 'generateDeploySql',
@@ -325,8 +456,8 @@ module.exports = {
     const targetDb = generateDbPairingId(
       extendDatabaseInfo(await this.structure({ conid: targetConid, database: targetDatabase }))
     );
-    // const sourceConnection = await connections.get({conid:sourceConid})
-    const connection = await connections.get({ conid: targetConid });
+    // const sourceConnection = await connections.getCore({conid:sourceConid})
+    const connection = await connections.getCore({ conid: targetConid });
     const driver = requireEngineDriver(connection);
     const targetDbPaired = matchPairedObjects(sourceDb, targetDb, dbDiffOptions);
     const diffRows = computeDbDiffRows(sourceDb, targetDbPaired, dbDiffOptions, driver);

packages/api/src/controllers/files.js

@@ -3,11 +3,12 @@ const fs = require('fs-extra');
 const path = require('path');
 const { filesdir, archivedir, resolveArchiveFolder, uploadsdir, appdir } = require('../utility/directories');
 const getChartExport = require('../utility/getChartExport');
-const hasPermission = require('../utility/hasPermission');
+const { hasPermission } = require('../utility/hasPermission');
 const socket = require('../utility/socket');
 const scheduler = require('./scheduler');
 const getDiagramExport = require('../utility/getDiagramExport');
 const apps = require('./apps');
+const getMapExport = require('../utility/getMapExport');
 
 function serialize(format, data) {
   if (format == 'text') return data;
@@ -23,8 +24,8 @@ function deserialize(format, text) {
 
 module.exports = {
   list_meta: true,
-  async list({ folder }) {
-    if (!hasPermission(`files/${folder}/read`)) return [];
+  async list({ folder }, req) {
+    if (!hasPermission(`files/${folder}/read`, req)) return [];
     const dir = path.join(filesdir(), folder);
     if (!(await fs.exists(dir))) return [];
     const files = (await fs.readdir(dir)).map(file => ({ folder, file }));
@@ -32,11 +33,11 @@ module.exports = {
   },
 
   listAll_meta: true,
-  async listAll() {
+  async listAll(_params, req) {
     const folders = await fs.readdir(filesdir());
     const res = [];
     for (const folder of folders) {
-      if (!hasPermission(`files/${folder}/read`)) continue;
+      if (!hasPermission(`files/${folder}/read`, req)) continue;
       const dir = path.join(filesdir(), folder);
       const files = (await fs.readdir(dir)).map(file => ({ folder, file }));
       res.push(...files);
@@ -45,31 +46,43 @@ module.exports = {
   },
 
   delete_meta: true,
-  async delete({ folder, file }) {
-    if (!hasPermission(`files/${folder}/write`)) return;
+  async delete({ folder, file }, req) {
+    if (!hasPermission(`files/${folder}/write`, req)) return false;
     await fs.unlink(path.join(filesdir(), folder, file));
-    socket.emitChanged(`files-changed-${folder}`);
+    socket.emitChanged(`files-changed`, { folder });
     socket.emitChanged(`all-files-changed`);
+    return true;
   },
 
   rename_meta: true,
-  async rename({ folder, file, newFile }) {
-    if (!hasPermission(`files/${folder}/write`)) return;
+  async rename({ folder, file, newFile }, req) {
+    if (!hasPermission(`files/${folder}/write`, req)) return false;
     await fs.rename(path.join(filesdir(), folder, file), path.join(filesdir(), folder, newFile));
-    socket.emitChanged(`files-changed-${folder}`);
+    socket.emitChanged(`files-changed`, { folder });
     socket.emitChanged(`all-files-changed`);
+    return true;
+  },
+
+  refresh_meta: true,
+  async refresh({ folders }, req) {
+    for (const folder of folders) {
+      socket.emitChanged(`files-changed`, { folder });
+      socket.emitChanged(`all-files-changed`);
+    }
+    return true;
   },
 
   copy_meta: true,
-  async copy({ folder, file, newFile }) {
-    if (!hasPermission(`files/${folder}/write`)) return;
+  async copy({ folder, file, newFile }, req) {
+    if (!hasPermission(`files/${folder}/write`, req)) return false;
     await fs.copyFile(path.join(filesdir(), folder, file), path.join(filesdir(), folder, newFile));
-    socket.emitChanged(`files-changed-${folder}`);
+    socket.emitChanged(`files-changed`, { folder });
     socket.emitChanged(`all-files-changed`);
+    return true;
   },
 
   load_meta: true,
-  async load({ folder, file, format }) {
+  async load({ folder, file, format }, req) {
     if (folder.startsWith('archive:')) {
       const text = await fs.readFile(path.join(resolveArchiveFolder(folder.substring('archive:'.length)), file), {
         encoding: 'utf-8',
@@ -81,33 +94,42 @@ module.exports = {
       });
       return deserialize(format, text);
     } else {
-      if (!hasPermission(`files/${folder}/read`)) return null;
+      if (!hasPermission(`files/${folder}/read`, req)) return null;
       const text = await fs.readFile(path.join(filesdir(), folder, file), { encoding: 'utf-8' });
       return deserialize(format, text);
     }
   },
 
+  loadFrom_meta: true,
+  async loadFrom({ filePath, format }, req) {
+    const text = await fs.readFile(filePath, { encoding: 'utf-8' });
+    return deserialize(format, text);
+  },
+
   save_meta: true,
-  async save({ folder, file, data, format }) {
+  async save({ folder, file, data, format }, req) {
     if (folder.startsWith('archive:')) {
+      if (!hasPermission(`archive/write`, req)) return false;
       const dir = resolveArchiveFolder(folder.substring('archive:'.length));
       await fs.writeFile(path.join(dir, file), serialize(format, data));
-      socket.emitChanged(`archive-files-changed-${folder.substring('archive:'.length)}`);
+      socket.emitChanged(`archive-files-changed`, { folder: folder.substring('archive:'.length) });
       return true;
     } else if (folder.startsWith('app:')) {
+      if (!hasPermission(`apps/write`, req)) return false;
       const app = folder.substring('app:'.length);
       await fs.writeFile(path.join(appdir(), app, file), serialize(format, data));
-      socket.emitChanged(`app-files-changed-${app}`);
+      socket.emitChanged(`app-files-changed`, { app });
+      socket.emitChanged('used-apps-changed');
       apps.emitChangedDbApp(folder);
       return true;
     } else {
-      if (!hasPermission(`files/${folder}/write`)) return false;
+      if (!hasPermission(`files/${folder}/write`, req)) return false;
       const dir = path.join(filesdir(), folder);
       if (!(await fs.exists(dir))) {
         await fs.mkdir(dir);
       }
       await fs.writeFile(path.join(dir, file), serialize(format, data));
-      socket.emitChanged(`files-changed-${folder}`);
+      socket.emitChanged(`files-changed`, { folder });
       socket.emitChanged(`all-files-changed`);
       if (folder == 'shell') {
         scheduler.reload();
@@ -122,8 +144,8 @@ module.exports = {
   },
 
   favorites_meta: true,
-  async favorites() {
-    if (!hasPermission(`files/favorites/read`)) return [];
+  async favorites(_params, req) {
+    if (!hasPermission(`files/favorites/read`, req)) return [];
     const dir = path.join(filesdir(), 'favorites');
     if (!(await fs.exists(dir))) return [];
     const files = await fs.readdir(dir);
@@ -141,8 +163,8 @@ module.exports = {
   },
 
   generateUploadsFile_meta: true,
-  async generateUploadsFile() {
-    const fileName = `${uuidv1()}.html`;
+  async generateUploadsFile({ extension }) {
+    const fileName = `${uuidv1()}.${extension || 'html'}`;
     return {
       fileName,
       filePath: path.join(uploadsdir(), fileName),
@@ -166,9 +188,35 @@ module.exports = {
     return true;
   },
 
+  exportMap_meta: true,
+  async exportMap({ filePath, geoJson }) {
+    await fs.writeFile(filePath, getMapExport(geoJson));
+    return true;
+  },
+
   exportDiagram_meta: true,
   async exportDiagram({ filePath, html, css, themeType, themeClassName }) {
     await fs.writeFile(filePath, getDiagramExport(html, css, themeType, themeClassName));
     return true;
   },
+
+  getFileRealPath_meta: true,
+  async getFileRealPath({ folder, file }, req) {
+    if (folder.startsWith('archive:')) {
+      if (!hasPermission(`archive/write`, req)) return false;
+      const dir = resolveArchiveFolder(folder.substring('archive:'.length));
+      return path.join(dir, file);
+    } else if (folder.startsWith('app:')) {
+      if (!hasPermission(`apps/write`, req)) return false;
+      const app = folder.substring('app:'.length);
+      return path.join(appdir(), app, file);
+    } else {
+      if (!hasPermission(`files/${folder}/write`, req)) return false;
+      const dir = path.join(filesdir(), folder);
+      if (!(await fs.exists(dir))) {
+        await fs.mkdir(dir);
+      }
+      return path.join(dir, file);
+    }
+  },
 };

packages/api/src/controllers/jsldata.js

@@ -1,11 +1,12 @@
+const { filterName } = require('dbgate-tools');
 const fs = require('fs');
 const lineReader = require('line-reader');
 const _ = require('lodash');
 const { __ } = require('lodash/fp');
 const DatastoreProxy = require('../utility/DatastoreProxy');
-const { saveFreeTableData } = require('../utility/freeTableStorage');
 const getJslFileName = require('../utility/getJslFileName');
 const JsonLinesDatastore = require('../utility/JsonLinesDatastore');
+const requirePluginFunction = require('../utility/requirePluginFunction');
 const socket = require('../utility/socket');
 
 function readFirstLine(file) {
@@ -98,37 +99,58 @@ module.exports = {
   //   return readerInfo;
   // },
 
-  async ensureDatastore(jslid) {
+  async ensureDatastore(jslid, formatterFunction) {
     let datastore = this.datastores[jslid];
-    if (!datastore) {
-      datastore = new JsonLinesDatastore(getJslFileName(jslid));
+    if (!datastore || datastore.formatterFunction != formatterFunction) {
+      if (datastore) {
+        datastore._closeReader();
+      }
+      datastore = new JsonLinesDatastore(getJslFileName(jslid), formatterFunction);
       // datastore = new DatastoreProxy(getJslFileName(jslid));
       this.datastores[jslid] = datastore;
     }
     return datastore;
   },
 
+  async closeDataStore(jslid) {
+    const datastore = this.datastores[jslid];
+    if (datastore) {
+      await datastore._closeReader();
+      delete this.datastores[jslid];
+    }
+  },
+
   getInfo_meta: true,
   async getInfo({ jslid }) {
     const file = getJslFileName(jslid);
-    const firstLine = await readFirstLine(file);
-    if (firstLine) {
-      const parsed = JSON.parse(firstLine);
-      if (parsed.__isStreamHeader) {
-        return parsed;
+    try {
+      const firstLine = await readFirstLine(file);
+      if (firstLine) {
+        const parsed = JSON.parse(firstLine);
+        if (parsed.__isStreamHeader) {
+          return parsed;
+        }
+        return {
+          __isStreamHeader: true,
+          __isDynamicStructure: true,
+        };
       }
-      return {
-        __isStreamHeader: true,
-        __isDynamicStructure: true,
-      };
+      return null;
+    } catch (err) {
+      return null;
     }
-    return null;
   },
 
   getRows_meta: true,
-  async getRows({ jslid, offset, limit, filters }) {
-    const datastore = await this.ensureDatastore(jslid);
-    return datastore.getRows(offset, limit, _.isEmpty(filters) ? null : filters);
+  async getRows({ jslid, offset, limit, filters, sort, formatterFunction }) {
+    const datastore = await this.ensureDatastore(jslid, formatterFunction);
+    return datastore.getRows(offset, limit, _.isEmpty(filters) ? null : filters, _.isEmpty(sort) ? null : sort);
+  },
+
+  exists_meta: true,
+  async exists({ jslid }) {
+    const fileName = getJslFileName(jslid);
+    return fs.existsSync(fileName);
   },
 
   getStats_meta: true,
@@ -144,6 +166,19 @@ module.exports = {
     return {};
   },
 
+  loadFieldValues_meta: true,
+  async loadFieldValues({ jslid, field, search, formatterFunction }) {
+    const datastore = await this.ensureDatastore(jslid, formatterFunction);
+    const res = new Set();
+    await datastore.enumRows(row => {
+      if (!filterName(search, row[field])) return true;
+      res.add(row[field]);
+      return res.size < 100;
+    });
+    // @ts-ignore
+    return [...res].map(value => ({ value }));
+  },
+
   async notifyChangedStats(stats) {
     // console.log('SENDING STATS', JSON.stringify(stats));
     const datastore = this.datastores[stats.jslid];
@@ -159,15 +194,100 @@ module.exports = {
     // }
   },
 
-  saveFreeTable_meta: true,
-  async saveFreeTable({ jslid, data }) {
-    saveFreeTableData(getJslFileName(jslid), data);
-    return true;
-  },
-
   saveText_meta: true,
   async saveText({ jslid, text }) {
     await fs.promises.writeFile(getJslFileName(jslid), text);
     return true;
   },
+
+  saveRows_meta: true,
+  async saveRows({ jslid, rows }) {
+    const fileStream = fs.createWriteStream(getJslFileName(jslid));
+    for (const row of rows) {
+      await fileStream.write(JSON.stringify(row) + '\n');
+    }
+    await fileStream.close();
+    return true;
+  },
+
+  extractTimelineChart_meta: true,
+  async extractTimelineChart({ jslid, timestampFunction, aggregateFunction, measures }) {
+    const timestamp = requirePluginFunction(timestampFunction);
+    const aggregate = requirePluginFunction(aggregateFunction);
+    const datastore = new JsonLinesDatastore(getJslFileName(jslid));
+    let mints = null;
+    let maxts = null;
+    // pass 1 - counts stats, time range
+    await datastore.enumRows(row => {
+      const ts = timestamp(row);
+      if (!mints || ts < mints) mints = ts;
+      if (!maxts || ts > maxts) maxts = ts;
+      return true;
+    });
+    const minTime = new Date(mints).getTime();
+    const maxTime = new Date(maxts).getTime();
+    const duration = maxTime - minTime;
+    const STEPS = 100;
+    let stepCount = duration > 100 * 1000 ? STEPS : Math.round((maxTime - minTime) / 1000);
+    if (stepCount < 2) {
+      stepCount = 2;
+    }
+    const stepDuration = duration / stepCount;
+    const labels = _.range(stepCount).map(i => new Date(minTime + stepDuration / 2 + stepDuration * i));
+
+    // const datasets = measures.map(m => ({
+    //   label: m.label,
+    //   data: Array(stepCount).fill(0),
+    // }));
+
+    const mproc = measures.map(m => ({
+      ...m,
+    }));
+
+    const data = Array(stepCount)
+      .fill(0)
+      .map(() => ({}));
+
+    // pass 2 - count measures
+    await datastore.enumRows(row => {
+      const ts = timestamp(row);
+      let part = Math.round((new Date(ts).getTime() - minTime) / stepDuration);
+      if (part < 0) part = 0;
+      if (part >= stepCount) part - stepCount - 1;
+      if (data[part]) {
+        data[part] = aggregate(data[part], row, stepDuration);
+      }
+      return true;
+    });
+
+    datastore._closeReader();
+
+    // const measureByField = _.fromPairs(measures.map((m, i) => [m.field, i]));
+
+    // for (let mindex = 0; mindex < measures.length; mindex++) {
+    //   for (let stepIndex = 0; stepIndex < stepCount; stepIndex++) {
+    //     const measure = measures[mindex];
+    //     if (measure.perSecond) {
+    //       datasets[mindex].data[stepIndex] /= stepDuration / 1000;
+    //     }
+    //     if (measure.perField) {
+    //       datasets[mindex].data[stepIndex] /= datasets[measureByField[measure.perField]].data[stepIndex];
+    //     }
+    //   }
+    // }
+
+    // for (let i = 0; i < measures.length; i++) {
+    //   if (measures[i].hidden) {
+    //     datasets[i] = null;
+    //   }
+    // }
+
+    return {
+      labels,
+      datasets: mproc.map(m => ({
+        label: m.label,
+        data: data.map(d => d[m.field] || 0),
+      })),
+    };
+  },
 };

packages/api/src/controllers/plugins.js

@@ -7,7 +7,7 @@ const socket = require('../utility/socket');
 const compareVersions = require('compare-versions');
 const requirePlugin = require('../shell/requirePlugin');
 const downloadPackage = require('../utility/downloadPackage');
-const hasPermission = require('../utility/hasPermission');
+const { hasPermission } = require('../utility/hasPermission');
 const _ = require('lodash');
 const packagedPluginsContent = require('../packagedPluginsContent');
 
@@ -73,7 +73,7 @@ module.exports = {
     const res = [];
     for (const packageName of _.union(files1, files2)) {
       if (packageName == 'dist') continue;
-      // if (!/^dbgate-plugin-.*$/.test(packageName)) continue;
+      if (!/^dbgate-plugin-.*$/.test(packageName)) continue;
       try {
         if (packagedContent && packagedContent[packageName]) {
           const manifest = {
@@ -89,6 +89,12 @@ module.exports = {
               encoding: 'utf-8',
             })
             .then(x => JSON.parse(x));
+          if (!manifest.keywords) {
+            continue;
+          }
+          if (!manifest.keywords.includes('dbgateplugin')) {
+            continue;
+          }
           const readmeFile = path.join(isPackaged ? packagedPluginsDir() : pluginsdir(), packageName, 'README.md');
           // @ts-ignore
           if (await fs.exists(readmeFile)) {
@@ -109,8 +115,8 @@ module.exports = {
   // },
 
   install_meta: true,
-  async install({ packageName }) {
-    if (!hasPermission(`plugins/install`)) return;
+  async install({ packageName }, req) {
+    if (!hasPermission(`plugins/install`, req)) return;
     const dir = path.join(pluginsdir(), packageName);
     // @ts-ignore
     if (!(await fs.exists(dir))) {
@@ -119,21 +125,23 @@ module.exports = {
     socket.emitChanged(`installed-plugins-changed`);
     // this.removedPlugins = this.removedPlugins.filter(x => x != packageName);
     // await this.saveRemovePlugins();
+    return true;
   },
 
   uninstall_meta: true,
-  async uninstall({ packageName }) {
-    if (!hasPermission(`plugins/install`)) return;
+  async uninstall({ packageName }, req) {
+    if (!hasPermission(`plugins/install`, req)) return;
     const dir = path.join(pluginsdir(), packageName);
     await fs.rmdir(dir, { recursive: true });
     socket.emitChanged(`installed-plugins-changed`);
     // this.removedPlugins.push(packageName);
-    await this.saveRemovePlugins();
+    // await this.saveRemovePlugins();
+    return true;
   },
 
   upgrade_meta: true,
-  async upgrade({ packageName }) {
-    if (!hasPermission(`plugins/install`)) return;
+  async upgrade({ packageName }, req) {
+    if (!hasPermission(`plugins/install`, req)) return;
     const dir = path.join(pluginsdir(), packageName);
     // @ts-ignore
     if (await fs.exists(dir)) {
@@ -142,6 +150,7 @@ module.exports = {
     }
 
     socket.emitChanged(`installed-plugins-changed`);
+    return true;
   },
 
   command_meta: true,
@@ -153,6 +162,7 @@ module.exports = {
   authTypes_meta: true,
   async authTypes({ engine }) {
     const packageName = extractPackageName(engine);
+    if (!packageName) return null;
     const content = requirePlugin(packageName);
     const driver = content.drivers.find(x => x.engine == engine);
     if (!driver || !driver.getAuthTypes) return null;

packages/api/src/controllers/runners.js

@@ -6,9 +6,17 @@ const byline = require('byline');
 const socket = require('../utility/socket');
 const { fork } = require('child_process');
 const { rundir, uploadsdir, pluginsdir, getPluginBackendPath, packagedPluginList } = require('../utility/directories');
-const { extractShellApiPlugins, extractShellApiFunctionName } = require('dbgate-tools');
+const {
+  extractShellApiPlugins,
+  extractShellApiFunctionName,
+  jsonScriptToJavascript,
+  getLogger,
+  safeJsonParse,
+} = require('dbgate-tools');
 const { handleProcessCommunication } = require('../utility/processComm');
 const processArgs = require('../utility/processArgs');
+const platformInfo = require('../utility/platformInfo');
+const logger = getLogger('runners');
 
 function extractPlugins(script) {
   const requireRegex = /\s*\/\/\s*@require\s+([^\s]+)\s*\n/g;
@@ -28,13 +36,14 @@ const requirePluginsTemplate = (plugins, isExport) =>
 
 const scriptTemplate = (script, isExport) => `
 const dbgateApi = require(${isExport ? `'dbgate-api'` : 'process.env.DBGATE_API'});
+const logger = dbgateApi.getLogger('script');
 dbgateApi.initializeApiEnvironment();
 ${requirePluginsTemplate(extractPlugins(script), isExport)}
 require=null;
 async function run() {
 ${script}
 await dbgateApi.finalizer.run();
-console.log('Finished job script');
+logger.info('Finished job script');
 }
 dbgateApi.runScript(run);
 `;
@@ -58,20 +67,23 @@ module.exports = {
   requests: {},
 
   dispatchMessage(runid, message) {
-    if (message) console.log('...', message.message);
-    if (_.isString(message)) {
-      socket.emit(`runner-info-${runid}`, {
-        message,
+    if (message) {
+      const json = safeJsonParse(message.message);
+
+      if (json) logger.log(json);
+      else logger.info(message.message);
+
+      const toEmit = {
         time: new Date(),
-        severity: 'info',
-      });
-    }
-    if (_.isPlainObject(message)) {
-      socket.emit(`runner-info-${runid}`, {
-        time: new Date(),
-        severity: 'info',
         ...message,
-      });
+        message: json ? json.msg : message.message,
+      };
+
+      if (json && json.level >= 50) {
+        toEmit.severity = 'error';
+      }
+
+      socket.emit(`runner-info-${runid}`, toEmit);
     }
   },
 
@@ -97,12 +109,15 @@ module.exports = {
     fs.writeFileSync(`${scriptFile}`, scriptText);
     fs.mkdirSync(directory);
     const pluginNames = _.union(fs.readdirSync(pluginsdir()), packagedPluginList);
-    console.log(`RUNNING SCRIPT ${scriptFile}`);
+    logger.info({ scriptFile }, 'Running script');
     // const subprocess = fork(scriptFile, ['--checkParent', '--max-old-space-size=8192'], {
     const subprocess = fork(
       scriptFile,
       [
         '--checkParent', // ...process.argv.slice(3)
+        '--is-forked-api',
+        '--process-display-name',
+        'script',
         ...processArgs.getPassArgs(),
       ],
       {
@@ -110,19 +125,20 @@ module.exports = {
         stdio: ['ignore', 'pipe', 'pipe', 'ipc'],
         env: {
           ...process.env,
-          DBGATE_API: global['API_PACKAGE'] || global['dbgateApiModulePath'] || process.argv[1],
+          DBGATE_API: global['API_PACKAGE'] || process.argv[1],
           ..._.fromPairs(pluginNames.map(name => [`PLUGIN_${_.camelCase(name)}`, getPluginBackendPath(name)])),
         },
       }
     );
-    const pipeDispatcher = severity => data =>
-      this.dispatchMessage(runid, { severity, message: data.toString().trim() });
+    const pipeDispatcher = severity => data => {
+      return this.dispatchMessage(runid, { severity, message: data.toString().trim() });
+    };
 
     byline(subprocess.stdout).on('data', pipeDispatcher('info'));
     byline(subprocess.stderr).on('data', pipeDispatcher('error'));
     subprocess.on('exit', code => {
       this.rejectRequest(runid, { message: 'No data retured, maybe input data source is too big' });
-      console.log('... EXIT process', code);
+      logger.info({ code, pid: subprocess.pid }, 'Exited process');
       socket.emit(`runner-done-${runid}`, code);
     });
     subprocess.on('error', error => {
@@ -150,6 +166,16 @@ module.exports = {
   start_meta: true,
   async start({ script }) {
     const runid = uuidv1();
+
+    if (script.type == 'json') {
+      const js = jsonScriptToJavascript(script);
+      return this.startCore(runid, scriptTemplate(js, false));
+    }
+
+    if (!platformInfo.allowShellScripting) {
+      return { errorMessage: 'Shell scripting is not allowed' };
+    }
+
     return this.startCore(runid, scriptTemplate(script, false));
   },
 

packages/api/src/controllers/scheduler.js

@@ -3,7 +3,10 @@ const fs = require('fs-extra');
 const path = require('path');
 const cron = require('node-cron');
 const runners = require('./runners');
-const hasPermission = require('../utility/hasPermission');
+const { hasPermission } = require('../utility/hasPermission');
+const { getLogger } = require('dbgate-tools');
+
+const logger = getLogger('scheduler');
 
 const scheduleRegex = /\s*\/\/\s*@schedule\s+([^\n]+)\n/;
 
@@ -21,13 +24,13 @@ module.exports = {
     if (!match) return;
     const pattern = match[1];
     if (!cron.validate(pattern)) return;
-    console.log(`Schedule script ${file} with pattern ${pattern}`);
+    logger.info(`Schedule script ${file} with pattern ${pattern}`);
     const task = cron.schedule(pattern, () => runners.start({ script: text }));
     this.tasks.push(task);
   },
 
-  async reload() {
-    if (!hasPermission('files/shell/read')) return;
+  async reload(_params, req) {
+    if (!hasPermission('files/shell/read', req)) return;
     const shellDir = path.join(filesdir(), 'shell');
     await this.unload();
     if (!(await fs.exists(shellDir))) return;

packages/api/src/controllers/serverConnections.js

@@ -1,29 +1,37 @@
 const connections = require('./connections');
 const socket = require('../utility/socket');
 const { fork } = require('child_process');
+const uuidv1 = require('uuid/v1');
 const _ = require('lodash');
 const AsyncLock = require('async-lock');
 const { handleProcessCommunication } = require('../utility/processComm');
 const lock = new AsyncLock();
 const config = require('./config');
 const processArgs = require('../utility/processArgs');
+const { testConnectionPermission } = require('../utility/hasPermission');
+const { MissingCredentialsError } = require('../utility/exceptions');
+const pipeForkLogs = require('../utility/pipeForkLogs');
+const { getLogger } = require('dbgate-tools');
+
+const logger = getLogger('serverConnection');
 
 module.exports = {
   opened: [],
   closed: {},
   lastPinged: {},
+  requests: {},
 
   handle_databases(conid, { databases }) {
     const existing = this.opened.find(x => x.conid == conid);
     if (!existing) return;
     existing.databases = databases;
-    socket.emitChanged(`database-list-changed-${conid}`);
+    socket.emitChanged(`database-list-changed`, { conid });
   },
   handle_version(conid, { version }) {
     const existing = this.opened.find(x => x.conid == conid);
     if (!existing) return;
     existing.version = version;
-    socket.emitChanged(`server-version-changed-${conid}`);
+    socket.emitChanged(`server-version-changed`, { conid });
   },
   handle_status(conid, { status }) {
     const existing = this.opened.find(x => x.conid == conid);
@@ -32,19 +40,37 @@ module.exports = {
     socket.emitChanged(`server-status-changed`);
   },
   handle_ping() {},
+  handle_response(conid, { msgid, ...response }) {
+    const [resolve, reject] = this.requests[msgid];
+    resolve(response);
+    delete this.requests[msgid];
+  },
 
   async ensureOpened(conid) {
     const res = await lock.acquire(conid, async () => {
       const existing = this.opened.find(x => x.conid == conid);
       if (existing) return existing;
-      const connection = await connections.get({ conid });
-      const subprocess = fork(global['API_PACKAGE'] || process.argv[1], [
-        '--is-forked-api',
-        '--start-process',
-        'serverConnectionProcess',
-        ...processArgs.getPassArgs(),
-        // ...process.argv.slice(3),
-      ]);
+      const connection = await connections.getCore({ conid });
+      if (!connection) {
+        throw new Error(`Connection with conid="${conid}" not fund`);
+      }
+      if (connection.passwordMode == 'askPassword' || connection.passwordMode == 'askUser') {
+        throw new MissingCredentialsError({ conid, passwordMode: connection.passwordMode });
+      }
+      const subprocess = fork(
+        global['API_PACKAGE'] || process.argv[1],
+        [
+          '--is-forked-api',
+          '--start-process',
+          'serverConnectionProcess',
+          ...processArgs.getPassArgs(),
+          // ...process.argv.slice(3),
+        ],
+        {
+          stdio: ['ignore', 'pipe', 'pipe', 'ipc'],
+        }
+      );
+      pipeForkLogs(subprocess);
       const newOpened = {
         conid,
         subprocess,
@@ -69,7 +95,7 @@ module.exports = {
         if (newOpened.disconnected) return;
         this.close(conid, false);
       });
-      subprocess.send({ msgtype: 'connect', ...connection, globalSettings: config.settingsValue });
+      subprocess.send({ msgtype: 'connect', ...connection, globalSettings: await config.getSettings() });
       return newOpened;
     });
     return res;
@@ -79,7 +105,13 @@ module.exports = {
     const existing = this.opened.find(x => x.conid == conid);
     if (existing) {
       existing.disconnected = true;
-      if (kill) existing.subprocess.kill();
+      if (kill) {
+        try {
+          existing.subprocess.kill();
+        } catch (err) {
+          logger.error({ err }, 'Error killing subprocess');
+        }
+      }
       this.opened = this.opened.filter(x => x.conid != conid);
       this.closed[conid] = {
         ...existing.status,
@@ -90,19 +122,23 @@ module.exports = {
   },
 
   disconnect_meta: true,
-  async disconnect({ conid }) {
+  async disconnect({ conid }, req) {
+    testConnectionPermission(conid, req);
     await this.close(conid, true);
     return { status: 'ok' };
   },
 
   listDatabases_meta: true,
-  async listDatabases({ conid }) {
+  async listDatabases({ conid }, req) {
+    if (!conid) return [];
+    testConnectionPermission(conid, req);
     const opened = await this.ensureOpened(conid);
     return opened.databases;
   },
 
   version_meta: true,
-  async version({ conid }) {
+  async version({ conid }, req) {
+    testConnectionPermission(conid, req);
     const opened = await this.ensureOpened(conid);
     return opened.version;
   },
@@ -116,23 +152,29 @@ module.exports = {
   },
 
   ping_meta: true,
-  async ping({ connections }) {
+  async ping({ conidArray }) {
     await Promise.all(
-      _.uniq(connections).map(async conid => {
+      _.uniq(conidArray).map(async conid => {
         const last = this.lastPinged[conid];
         if (last && new Date().getTime() - last < 30 * 1000) {
           return Promise.resolve();
         }
         this.lastPinged[conid] = new Date().getTime();
         const opened = await this.ensureOpened(conid);
-        opened.subprocess.send({ msgtype: 'ping' });
+        try {
+          opened.subprocess.send({ msgtype: 'ping' });
+        } catch (err) {
+          logger.error({ err }, 'Error calling ping');
+          this.close(conid);
+        }
       })
     );
     return { status: 'ok' };
   },
 
   refresh_meta: true,
-  async refresh({ conid, keepOpen }) {
+  async refresh({ conid, keepOpen }, req) {
+    testConnectionPermission(conid, req);
     if (!keepOpen) this.close(conid);
 
     await this.ensureOpened(conid);
@@ -140,9 +182,62 @@ module.exports = {
   },
 
   createDatabase_meta: true,
-  async createDatabase({ conid, name }) {
+  async createDatabase({ conid, name }, req) {
+    testConnectionPermission(conid, req);
     const opened = await this.ensureOpened(conid);
+    if (opened.connection.isReadOnly) return false;
     opened.subprocess.send({ msgtype: 'createDatabase', name });
     return { status: 'ok' };
   },
+
+  dropDatabase_meta: true,
+  async dropDatabase({ conid, name }, req) {
+    testConnectionPermission(conid, req);
+    const opened = await this.ensureOpened(conid);
+    if (opened.connection.isReadOnly) return false;
+    opened.subprocess.send({ msgtype: 'dropDatabase', name });
+    return { status: 'ok' };
+  },
+
+  sendRequest(conn, message) {
+    const msgid = uuidv1();
+    const promise = new Promise((resolve, reject) => {
+      this.requests[msgid] = [resolve, reject];
+      try {
+        conn.subprocess.send({ msgid, ...message });
+      } catch (err) {
+        logger.error({ err }, 'Error sending request');
+        this.close(conn.conid);
+      }
+    });
+    return promise;
+  },
+
+  async loadDataCore(msgtype, { conid, ...args }, req) {
+    testConnectionPermission(conid, req);
+    const opened = await this.ensureOpened(conid);
+    const res = await this.sendRequest(opened, { msgtype, ...args });
+    if (res.errorMessage) {
+      console.error(res.errorMessage);
+
+      return {
+        errorMessage: res.errorMessage,
+      };
+    }
+    return res.result || null;
+  },
+
+  serverSummary_meta: true,
+  async serverSummary({ conid }, req) {
+    testConnectionPermission(conid, req);
+    return this.loadDataCore('serverSummary', { conid });
+  },
+
+  summaryCommand_meta: true,
+  async summaryCommand({ conid, command, row }, req) {
+    testConnectionPermission(conid, req);
+    const opened = await this.ensureOpened(conid);
+    if (opened.connection.isReadOnly) return false;
+    return this.loadDataCore('summaryCommand', { conid, command, row });
+  },
 };

packages/api/src/controllers/sessions.js

@@ -4,8 +4,15 @@ const connections = require('./connections');
 const socket = require('../utility/socket');
 const { fork } = require('child_process');
 const jsldata = require('./jsldata');
+const path = require('path');
 const { handleProcessCommunication } = require('../utility/processComm');
 const processArgs = require('../utility/processArgs');
+const { appdir } = require('../utility/directories');
+const { getLogger } = require('dbgate-tools');
+const pipeForkLogs = require('../utility/pipeForkLogs');
+const config = require('./config');
+
+const logger = getLogger('sessions');
 
 module.exports = {
   /** @type {import('dbgate-types').OpenedSession[]} */
@@ -46,9 +53,18 @@ module.exports = {
     this.dispatchMessage(sesid, info);
   },
 
-  handle_done(sesid) {
+  handle_done(sesid, props) {
     socket.emit(`session-done-${sesid}`);
-    this.dispatchMessage(sesid, 'Query execution finished');
+    if (!props.skipFinishedMessage) {
+      this.dispatchMessage(sesid, 'Query execution finished');
+    }
+    const session = this.opened.find(x => x.sesid == sesid);
+    if (session.loadingReader_jslid) {
+      socket.emit(`session-jslid-done-${session.loadingReader_jslid}`);
+    }
+    if (session.killOnDone) {
+      this.kill({ sesid });
+    }
   },
 
   handle_recordset(sesid, props) {
@@ -60,19 +76,31 @@ module.exports = {
     jsldata.notifyChangedStats(stats);
   },
 
+  handle_initializeFile(sesid, props) {
+    const { jslid } = props;
+    socket.emit(`session-initialize-file-${jslid}`);
+  },
+
   handle_ping() {},
 
   create_meta: true,
   async create({ conid, database }) {
     const sesid = uuidv1();
-    const connection = await connections.get({ conid });
-    const subprocess = fork(global['API_PACKAGE'] || process.argv[1], [
-      '--is-forked-api',
-      '--start-process',
-      'sessionProcess',
-      ...processArgs.getPassArgs(),
-      // ...process.argv.slice(3),
-    ]);
+    const connection = await connections.getCore({ conid });
+    const subprocess = fork(
+      global['API_PACKAGE'] || process.argv[1],
+      [
+        '--is-forked-api',
+        '--start-process',
+        'sessionProcess',
+        ...processArgs.getPassArgs(),
+        // ...process.argv.slice(3),
+      ],
+      {
+        stdio: ['ignore', 'pipe', 'pipe', 'ipc'],
+      }
+    );
+    pipeForkLogs(subprocess);
     const newOpened = {
       conid,
       database,
@@ -87,7 +115,18 @@ module.exports = {
       if (handleProcessCommunication(message, subprocess)) return;
       this[`handle_${msgtype}`](sesid, message);
     });
-    subprocess.send({ msgtype: 'connect', ...connection, database });
+    subprocess.on('exit', () => {
+      this.opened = this.opened.filter(x => x.sesid != sesid);
+      this.dispatchMessage(sesid, 'Query session closed');
+      socket.emit(`session-closed-${sesid}`);
+    });
+
+    subprocess.send({
+      msgtype: 'connect',
+      ...connection,
+      database,
+      globalSettings: await config.getSettings(),
+    });
     return _.pick(newOpened, ['conid', 'database', 'sesid']);
   },
 
@@ -98,13 +137,61 @@ module.exports = {
       throw new Error('Invalid session');
     }
 
-    console.log(`Processing query, sesid=${sesid}, sql=${sql}`);
+    logger.info({ sesid, sql }, 'Processing query');
     this.dispatchMessage(sesid, 'Query execution started');
     session.subprocess.send({ msgtype: 'executeQuery', sql });
 
     return { state: 'ok' };
   },
 
+  executeReader_meta: true,
+  async executeReader({ conid, database, sql, queryName, appFolder }) {
+    const { sesid } = await this.create({ conid, database });
+    const session = this.opened.find(x => x.sesid == sesid);
+    session.killOnDone = true;
+    const jslid = uuidv1();
+    session.loadingReader_jslid = jslid;
+    const fileName = queryName && appFolder ? path.join(appdir(), appFolder, `${queryName}.query.sql`) : null;
+
+    session.subprocess.send({ msgtype: 'executeReader', sql, fileName, jslid });
+
+    return { jslid };
+  },
+
+  stopLoadingReader_meta: true,
+  async stopLoadingReader({ jslid }) {
+    const session = this.opened.find(x => x.loadingReader_jslid == jslid);
+    if (session) {
+      this.kill({ sesid: session.sesid });
+    }
+    return true;
+  },
+
+  startProfiler_meta: true,
+  async startProfiler({ sesid }) {
+    const jslid = uuidv1();
+    const session = this.opened.find(x => x.sesid == sesid);
+    if (!session) {
+      throw new Error('Invalid session');
+    }
+
+    logger.info({ sesid }, 'Starting profiler');
+    session.loadingReader_jslid = jslid;
+    session.subprocess.send({ msgtype: 'startProfiler', jslid });
+
+    return { state: 'ok', jslid };
+  },
+
+  stopProfiler_meta: true,
+  async stopProfiler({ sesid }) {
+    const session = this.opened.find(x => x.sesid == sesid);
+    if (!session) {
+      throw new Error('Invalid session');
+    }
+    session.subprocess.send({ msgtype: 'stopProfiler' });
+    return { state: 'ok' };
+  },
+
   // cancel_meta: true,
   // async cancel({ sesid }) {
   //   const session = this.opened.find((x) => x.sesid == sesid);
@@ -126,6 +213,17 @@ module.exports = {
     return { state: 'ok' };
   },
 
+  ping_meta: true,
+  async ping({ sesid }) {
+    const session = this.opened.find(x => x.sesid == sesid);
+    if (!session) {
+      throw new Error('Invalid session');
+    }
+    session.subprocess.send({ msgtype: 'ping' });
+
+    return { state: 'ok' };
+  },
+
   // runCommand_meta: true,
   // async runCommand({ conid, database, sql }) {
   //   console.log(`Running SQL command , conid=${conid}, database=${database}, sql=${sql}`);

packages/api/src/controllers/uploads.js

@@ -1,6 +1,8 @@
 const path = require('path');
 const { uploadsdir } = require('../utility/directories');
 const uuidv1 = require('uuid/v1');
+const { getLogger } = require('dbgate-tools');
+const logger = getLogger('uploads');
 
 module.exports = {
   upload_meta: {
@@ -15,7 +17,7 @@ module.exports = {
     }
     const uploadName = uuidv1();
     const filePath = path.join(uploadsdir(), uploadName);
-    console.log(`Uploading file ${data.name}, size=${data.size}`);
+    logger.info(`Uploading file ${data.name}, size=${data.size}`);
 
     data.mv(filePath, () => {
       res.json({

packages/api/src/currentVersion.js

@@ -1,5 +1,5 @@
 
 module.exports = { 
-  version: '4.1.1',
+  version: '5.0.0-alpha.1',
   buildTime: '2021-04-17T07:22:49.702Z'
 };

packages/api/src/index.js

@@ -1,5 +1,96 @@
-const shell = require('./shell');
+const { setLogger, getLogger, setLoggerName } = require('dbgate-tools');
 const processArgs = require('./utility/processArgs');
+const fs = require('fs');
+const moment = require('moment');
+const path = require('path');
+const { logsdir, setLogsFilePath, getLogsFilePath } = require('./utility/directories');
+const { createLogger } = require('pinomin');
+
+if (processArgs.startProcess) {
+  setLoggerName(processArgs.startProcess.replace(/Process$/, ''));
+}
+if (processArgs.processDisplayName) {
+  setLoggerName(processArgs.processDisplayName);
+}
+
+// function loadLogsContent(maxLines) {
+//   const text = fs.readFileSync(getLogsFilePath(), { encoding: 'utf8' });
+//   if (maxLines) {
+//     const lines = text
+//       .split('\n')
+//       .map(x => x.trim())
+//       .filter(x => x);
+//     return lines.slice(-maxLines).join('\n');
+//   }
+//   return text;
+// }
+
+function configureLogger() {
+  const logsFilePath = path.join(logsdir(), `${moment().format('YYYY-MM-DD-HH-mm')}-${process.pid}.ndjson`);
+  setLogsFilePath(logsFilePath);
+  setLoggerName('main');
+
+  const logger = createLogger({
+    base: { pid: process.pid },
+    targets: [
+      {
+        type: 'console',
+        // @ts-ignore
+        level: process.env.CONSOLE_LOG_LEVEL || process.env.LOG_LEVEL || 'info',
+      },
+      {
+        type: 'stream',
+        // @ts-ignore
+        level: process.env.FILE_LOG_LEVEL || process.env.LOG_LEVEL || 'info',
+        stream: fs.createWriteStream(logsFilePath, { flags: 'a' }),
+      },
+    ],
+  });
+
+  // const streams = [];
+  // if (!platformInfo.isElectron) {
+  //   streams.push({
+  //     stream: process.stdout,
+  //     level: process.env.CONSOLE_LOG_LEVEL || process.env.LOG_LEVEL || 'info',
+  //   });
+  // }
+
+  // streams.push({
+  //   stream: fs.createWriteStream(logsFilePath),
+  //   level: process.env.FILE_LOG_LEVEL || process.env.LOG_LEVEL || 'info',
+  // });
+
+  // let logger = pinoms({
+  //   redact: { paths: ['hostname'], remove: true },
+  //   streams,
+  // });
+
+  // // @ts-ignore
+  // let logger = pino({
+  //   redact: { paths: ['hostname'], remove: true },
+  //   transport: {
+  //     targets: [
+  //       {
+  //         level: process.env.CONSOLE_LOG_LEVEL || process.env.LOG_LEVEL || 'info',
+  //         target: 'pino/file',
+  //       },
+  //       {
+  //         level: process.env.FILE_LOG_LEVEL || process.env.LOG_LEVEL || 'info',
+  //         target: 'pino/file',
+  //         options: { destination: logsFilePath },
+  //       },
+  //     ],
+  //   },
+  // });
+
+  setLogger(logger);
+}
+
+if (processArgs.listenApi) {
+  configureLogger();
+}
+
+const shell = require('./shell');
 const dbgateTools = require('dbgate-tools');
 
 global['DBGATE_TOOLS'] = dbgateTools;
@@ -8,13 +99,17 @@ if (processArgs.startProcess) {
   const proc = require('./proc');
   const module = proc[processArgs.startProcess];
   module.start();
-} else if (!processArgs.checkParent && !global['API_PACKAGE'] && !global['dbgateApiModulePath']) {
-  const main = require('./main');
+}
 
+if (processArgs.listenApi) {
+  const main = require('./main');
   main.start();
 }
 
 module.exports = {
   ...shell,
+  getLogger,
+  configureLogger,
+  // loadLogsContent,
   getMainModule: () => require('./main'),
 };

packages/api/src/main.js

@@ -20,15 +20,22 @@ const jsldata = require('./controllers/jsldata');
 const config = require('./controllers/config');
 const archive = require('./controllers/archive');
 const apps = require('./controllers/apps');
+const auth = require('./controllers/auth');
 const uploads = require('./controllers/uploads');
 const plugins = require('./controllers/plugins');
 const files = require('./controllers/files');
 const scheduler = require('./controllers/scheduler');
 const queryHistory = require('./controllers/queryHistory');
+const onFinished = require('on-finished');
 
 const { rundir } = require('./utility/directories');
 const platformInfo = require('./utility/platformInfo');
 const getExpressPath = require('./utility/getExpressPath');
+const { getLogins } = require('./utility/hasPermission');
+const _ = require('lodash');
+const { getLogger } = require('dbgate-tools');
+
+const logger = getLogger('main');
 
 function start() {
   // console.log('process.argv', process.argv);
@@ -37,12 +44,11 @@ function start() {
 
   const server = http.createServer(app);
 
-  if (process.env.LOGIN && process.env.PASSWORD) {
+  const logins = getLogins();
+  if (logins && process.env.BASIC_AUTH) {
     app.use(
       basicAuth({
-        users: {
-          [process.env.LOGIN]: process.env.PASSWORD,
-        },
+        users: _.fromPairs(logins.map(x => [x.login, x.password])),
         challenge: true,
         realm: 'DbGate Web App',
       })
@@ -51,17 +57,40 @@ function start() {
 
   app.use(cors());
 
+  if (platformInfo.isDocker) {
+    // server static files inside docker container
+    app.use(getExpressPath('/'), express.static('/home/dbgate-docker/public'));
+  } else if (platformInfo.isNpmDist) {
+    app.use(getExpressPath('/'), express.static(path.join(__dirname, '../../dbgate-web/public')));
+  } else if (process.env.DEVWEB) {
+    // console.log('__dirname', __dirname);
+    // console.log(path.join(__dirname, '../../web/public/build'));
+    app.use(getExpressPath('/'), express.static(path.join(__dirname, '../../web/public')));
+  } else {
+    app.get(getExpressPath('/'), (req, res) => {
+      res.send('DbGate API');
+    });
+  }
+
+  if (auth.shouldAuthorizeApi()) {
+    app.use(auth.authMiddleware);
+  }
+
   app.get(getExpressPath('/stream'), async function (req, res) {
     res.set({
       'Cache-Control': 'no-cache',
       'Content-Type': 'text/event-stream',
+      'X-Accel-Buffering': 'no',
       Connection: 'keep-alive',
     });
     res.flushHeaders();
 
     // Tell the client to retry every 10 seconds if connectivity is lost
     res.write('retry: 10000\n\n');
-    socket.setSseResponse(res);
+    socket.addSseResponse(res);
+    onFinished(req, () => {
+      socket.removeSseResponse(res);
+    });
   });
 
   app.use(bodyParser.json({ limit: '50mb' }));
@@ -82,37 +111,38 @@ function start() {
   app.use(getExpressPath('/runners/data'), express.static(rundir()));
 
   if (platformInfo.isDocker) {
-    // server static files inside docker container
-    app.use(getExpressPath('/'), express.static('/home/dbgate-docker/public'));
-  } else {
-    if (!platformInfo.isNpmDist) {
-      app.get(getExpressPath('/'), (req, res) => {
-        res.send('DbGate API');
-      });
-    }
-  }
-
-  if (platformInfo.isNpmDist) {
-    app.use(getExpressPath('/'), express.static(path.join(__dirname, '../../dbgate-web/public')));
-    getPort({ port: 5000 }).then(port => {
+    const port = process.env.PORT || 3000;
+    logger.info(`DbGate API listening on port ${port} (docker build)`);
+    server.listen(port);
+  } else if (platformInfo.isNpmDist) {
+    getPort({
+      port: parseInt(
+        // @ts-ignore
+        process.env.PORT || 3000
+      ),
+    }).then(port => {
       server.listen(port, () => {
-        console.log(`DbGate API listening on port ${port}`);
+        logger.info(`DbGate API listening on port ${port} (NPM build)`);
       });
     });
+  } else if (process.env.DEVWEB) {
+    const port = process.env.PORT || 3000;
+    logger.info(`DbGate API & web listening on port ${port} (dev web build)`);
+    server.listen(port);
   } else {
     const port = process.env.PORT || 3000;
-    console.log('DbGate API listening on port', port);
+    logger.info(`DbGate API listening on port ${port} (dev API build)`);
     server.listen(port);
   }
 
   function shutdown() {
-    console.log('\nShutting down DbGate API server');
+    logger.info('\nShutting down DbGate API server');
     server.close(() => {
-      console.log('Server shut down, terminating');
+      logger.info('Server shut down, terminating');
       process.exit(0);
     });
     setTimeout(() => {
-      console.log('Server close timeout, terminating');
+      logger.info('Server close timeout, terminating');
       process.exit(0);
     }, 1000);
   }
@@ -138,10 +168,11 @@ function useAllControllers(app, electron) {
   useController(app, electron, '/scheduler', scheduler);
   useController(app, electron, '/query-history', queryHistory);
   useController(app, electron, '/apps', apps);
+  useController(app, electron, '/auth', auth);
 }
 
-function initializeElectronSender(electronSender) {
+function setElectronSender(electronSender) {
   socket.setElectronSender(electronSender);
 }
 
-module.exports = { start, useAllControllers, initializeElectronSender };
+module.exports = { start, useAllControllers, setElectronSender, configController: config };

packages/api/src/proc/connectProcess.js

@@ -20,7 +20,7 @@ function start() {
     if (handleProcessCommunication(connection)) return;
     try {
       const driver = requireEngineDriver(connection);
-      const conn = await connectUtility(driver, connection);
+      const conn = await connectUtility(driver, connection, 'app');
       const res = await driver.getVersion(conn);
       process.send({ msgtype: 'connected', ...res });
     } catch (e) {

packages/api/src/proc/databaseConnectionProcess.js

@@ -1,12 +1,15 @@
 const stableStringify = require('json-stable-stringify');
 const { splitQuery } = require('dbgate-query-splitter');
 const childProcessChecker = require('../utility/childProcessChecker');
-const { extractBoolSettingsValue, extractIntSettingsValue } = require('dbgate-tools');
+const { extractBoolSettingsValue, extractIntSettingsValue, getLogger } = require('dbgate-tools');
 const requireEngineDriver = require('../utility/requireEngineDriver');
 const connectUtility = require('../utility/connectUtility');
 const { handleProcessCommunication } = require('../utility/processComm');
 const { SqlGenerator } = require('dbgate-tools');
 const generateDeploySql = require('../shell/generateDeploySql');
+const { dumpSqlSelect } = require('dbgate-sqltree');
+
+const logger = getLogger('dbconnProcess');
 
 let systemConnection;
 let storedConnection;
@@ -14,6 +17,7 @@ let afterConnectCallbacks = [];
 let afterAnalyseCallbacks = [];
 let analysedStructure = null;
 let lastPing = null;
+let lastStatusString = null;
 let lastStatus = null;
 let analysedTime = 0;
 let serverVersion;
@@ -77,21 +81,24 @@ async function handleIncrementalRefresh(forceSend) {
   resolveAnalysedPromises();
 }
 
-function handleSyncModel() {
+function handleSyncModel({ isFullRefresh }) {
   if (loadingModel) return;
-  handleIncrementalRefresh();
+  if (isFullRefresh) handleFullRefresh();
+  else handleIncrementalRefresh();
 }
 
 function setStatus(status) {
-  const statusString = stableStringify(status);
-  if (lastStatus != statusString) {
-    process.send({ msgtype: 'status', status: { ...status, counter: getStatusCounter() } });
-    lastStatus = statusString;
+  const newStatus = { ...lastStatus, ...status };
+  const statusString = stableStringify(newStatus);
+  if (lastStatusString != statusString) {
+    process.send({ msgtype: 'status', status: { ...newStatus, counter: getStatusCounter() } });
+    lastStatusString = statusString;
+    lastStatus = newStatus;
   }
 }
 
 function setStatusName(name) {
-  setStatus({ name });
+  setStatus({ name, message: null });
 }
 
 async function readVersion() {
@@ -107,7 +114,8 @@ async function handleConnect({ connection, structure, globalSettings }) {
 
   if (!structure) setStatusName('pending');
   const driver = requireEngineDriver(storedConnection);
-  systemConnection = await checkedAsyncCall(connectUtility(driver, storedConnection));
+  systemConnection = await checkedAsyncCall(connectUtility(driver, storedConnection, 'app'));
+  systemConnection.feedback = feedback => setStatus({ feedback });
   await checkedAsyncCall(readVersion());
   if (structure) {
     analysedStructure = structure;
@@ -150,36 +158,92 @@ function resolveAnalysedPromises() {
   afterAnalyseCallbacks = [];
 }
 
-async function handleRunScript({ msgid, sql }) {
+async function handleRunScript({ msgid, sql, useTransaction }, skipReadonlyCheck = false) {
   await waitConnected();
   const driver = requireEngineDriver(storedConnection);
   try {
-    await driver.script(systemConnection, sql);
+    if (!skipReadonlyCheck) ensureExecuteCustomScript(driver);
+    await driver.script(systemConnection, sql, { useTransaction });
     process.send({ msgtype: 'response', msgid });
   } catch (err) {
     process.send({ msgtype: 'response', msgid, errorMessage: err.message });
   }
 }
 
-async function handleQueryData({ msgid, sql }) {
+async function handleQueryData({ msgid, sql }, skipReadonlyCheck = false) {
   await waitConnected();
   const driver = requireEngineDriver(storedConnection);
   try {
+    if (!skipReadonlyCheck) ensureExecuteCustomScript(driver);
+    // console.log(sql);
     const res = await driver.query(systemConnection, sql);
     process.send({ msgtype: 'response', msgid, ...res });
+  } catch (err) {
+    process.send({ msgtype: 'response', msgid, errorMessage: err.message || 'Error executing SQL script' });
+  }
+}
+
+async function handleSqlSelect({ msgid, select }) {
+  const driver = requireEngineDriver(storedConnection);
+  const dmp = driver.createDumper();
+  dumpSqlSelect(dmp, select);
+  return handleQueryData({ msgid, sql: dmp.s }, true);
+}
+
+async function handleDriverDataCore(msgid, callMethod) {
+  await waitConnected();
+  const driver = requireEngineDriver(storedConnection);
+  try {
+    const result = await callMethod(driver);
+    process.send({ msgtype: 'response', msgid, result });
   } catch (err) {
     process.send({ msgtype: 'response', msgid, errorMessage: err.message });
   }
 }
 
 async function handleCollectionData({ msgid, options }) {
-  await waitConnected();
-  const driver = requireEngineDriver(storedConnection);
-  try {
-    const result = await driver.readCollection(systemConnection, options);
-    process.send({ msgtype: 'response', msgid, result });
-  } catch (err) {
-    process.send({ msgtype: 'response', msgid, errorMessage: err.message });
+  return handleDriverDataCore(msgid, driver => driver.readCollection(systemConnection, options));
+}
+
+async function handleLoadKeys({ msgid, root, filter }) {
+  return handleDriverDataCore(msgid, driver => driver.loadKeys(systemConnection, root, filter));
+}
+
+async function handleExportKeys({ msgid, options }) {
+  return handleDriverDataCore(msgid, driver => driver.exportKeys(systemConnection, options));
+}
+
+async function handleLoadKeyInfo({ msgid, key }) {
+  return handleDriverDataCore(msgid, driver => driver.loadKeyInfo(systemConnection, key));
+}
+
+async function handleCallMethod({ msgid, method, args }) {
+  return handleDriverDataCore(msgid, driver => {
+    if (storedConnection.isReadOnly) {
+      throw new Error('Connection is read only, cannot call custom methods');
+    }
+
+    ensureExecuteCustomScript(driver);
+    return driver.callMethod(systemConnection, method, args);
+  });
+}
+
+async function handleLoadKeyTableRange({ msgid, key, cursor, count }) {
+  return handleDriverDataCore(msgid, driver => driver.loadKeyTableRange(systemConnection, key, cursor, count));
+}
+
+async function handleLoadFieldValues({ msgid, schemaName, pureName, field, search }) {
+  return handleDriverDataCore(msgid, driver =>
+    driver.loadFieldValues(systemConnection, { schemaName, pureName }, field, search)
+  );
+}
+
+function ensureExecuteCustomScript(driver) {
+  if (driver.readOnlySessions) {
+    return;
+  }
+  if (storedConnection.isReadOnly) {
+    throw new Error('Connection is read only');
   }
 }
 
@@ -187,6 +251,7 @@ async function handleUpdateCollection({ msgid, changeSet }) {
   await waitConnected();
   const driver = requireEngineDriver(storedConnection);
   try {
+    ensureExecuteCustomScript(driver);
     const result = await driver.updateCollection(systemConnection, changeSet);
     process.send({ msgtype: 'response', msgid, result });
   } catch (err) {
@@ -206,7 +271,7 @@ async function handleSqlPreview({ msgid, objects, options }) {
     process.send({ msgtype: 'response', msgid, sql: dmp.s, isTruncated: generator.isTruncated });
     if (generator.isUnhandledException) {
       setTimeout(() => {
-        console.log('Exiting because of unhandled exception');
+        logger.error('Exiting because of unhandled exception');
         process.exit(0);
       }, 500);
     }
@@ -248,10 +313,17 @@ const messageHandlers = {
   runScript: handleRunScript,
   updateCollection: handleUpdateCollection,
   collectionData: handleCollectionData,
+  loadKeys: handleLoadKeys,
+  loadKeyInfo: handleLoadKeyInfo,
+  callMethod: handleCallMethod,
+  loadKeyTableRange: handleLoadKeyTableRange,
   sqlPreview: handleSqlPreview,
   ping: handlePing,
   syncModel: handleSyncModel,
   generateDeploySql: handleGenerateDeploySql,
+  loadFieldValues: handleLoadFieldValues,
+  sqlSelect: handleSqlSelect,
+  exportKeys: handleExportKeys,
   // runCommand: handleRunCommand,
 };
 
@@ -265,19 +337,19 @@ function start() {
 
   setInterval(() => {
     const time = new Date().getTime();
-    if (time - lastPing > 120 * 1000) {
-      console.log('Database connection not alive, exiting');
+    if (time - lastPing > 40 * 1000) {
+      logger.info('Database connection not alive, exiting');
       process.exit(0);
     }
-  }, 60 * 1000);
+  }, 10 * 1000);
 
   process.on('message', async message => {
     if (handleProcessCommunication(message)) return;
     try {
       await handleMessage(message);
-    } catch (e) {
-      console.error('Error in DB connection', e);
-      process.send({ msgtype: 'error', error: e.message });
+    } catch (err) {
+      logger.error({ err }, 'Error in DB connection');
+      process.send({ msgtype: 'error', error: err.message });
     }
   });
 }

packages/api/src/proc/index.js

@@ -3,6 +3,7 @@ const databaseConnectionProcess = require('./databaseConnectionProcess');
 const serverConnectionProcess = require('./serverConnectionProcess');
 const sessionProcess = require('./sessionProcess');
 const jslDatastoreProcess = require('./jslDatastoreProcess');
+const sshForwardProcess = require('./sshForwardProcess');
 
 module.exports = {
   connectProcess,
@@ -10,4 +11,5 @@ module.exports = {
   serverConnectionProcess,
   sessionProcess,
   jslDatastoreProcess,
+  sshForwardProcess,
 };

packages/api/src/proc/serverConnectionProcess.js

@@ -1,16 +1,17 @@
 const stableStringify = require('json-stable-stringify');
-const { extractBoolSettingsValue, extractIntSettingsValue } = require('dbgate-tools');
+const { extractBoolSettingsValue, extractIntSettingsValue, getLogger } = require('dbgate-tools');
 const childProcessChecker = require('../utility/childProcessChecker');
 const requireEngineDriver = require('../utility/requireEngineDriver');
-const { decryptConnection } = require('../utility/crypting');
 const connectUtility = require('../utility/connectUtility');
 const { handleProcessCommunication } = require('../utility/processComm');
+const logger = getLogger('srvconnProcess');
 
 let systemConnection;
 let storedConnection;
 let lastDatabases = null;
 let lastStatus = null;
 let lastPing = null;
+let afterConnectCallbacks = [];
 
 async function handleRefresh() {
   const driver = requireEngineDriver(storedConnection);
@@ -58,11 +59,14 @@ async function handleConnect(connection) {
 
   const driver = requireEngineDriver(storedConnection);
   try {
-    systemConnection = await connectUtility(driver, storedConnection);
+    systemConnection = await connectUtility(driver, storedConnection, 'app');
     readVersion();
     handleRefresh();
     if (extractBoolSettingsValue(globalSettings, 'connection.autoRefresh', false)) {
-      setInterval(handleRefresh, extractIntSettingsValue(globalSettings, 'connection.autoRefreshInterval', 30, 5, 3600) * 1000);
+      setInterval(
+        handleRefresh,
+        extractIntSettingsValue(globalSettings, 'connection.autoRefreshInterval', 30, 5, 3600) * 1000
+      );
     }
   } catch (err) {
     setStatus({
@@ -72,28 +76,64 @@ async function handleConnect(connection) {
     // console.error(err);
     setTimeout(() => process.exit(1), 1000);
   }
+
+  for (const [resolve] of afterConnectCallbacks) {
+    resolve();
+  }
+  afterConnectCallbacks = [];
+}
+
+function waitConnected() {
+  if (systemConnection) return Promise.resolve();
+  return new Promise((resolve, reject) => {
+    afterConnectCallbacks.push([resolve, reject]);
+  });
 }
 
 function handlePing() {
   lastPing = new Date().getTime();
 }
 
-async function handleCreateDatabase({ name }) {
+async function handleDatabaseOp(op, { name }) {
   const driver = requireEngineDriver(storedConnection);
-  systemConnection = await connectUtility(driver, storedConnection);
-  console.log(`RUNNING SCRIPT: CREATE DATABASE ${driver.dialect.quoteIdentifier(name)}`);
-  if (driver.createDatabase) {
-    await driver.createDatabase(systemConnection, name);
+  systemConnection = await connectUtility(driver, storedConnection, 'app');
+  if (driver[op]) {
+    await driver[op](systemConnection, name);
   } else {
-    await driver.query(systemConnection, `CREATE DATABASE ${driver.dialect.quoteIdentifier(name)}`);
+    const dmp = driver.createDumper();
+    dmp[op](name);
+    logger.info({ sql: dmp.s }, 'Running script');
+    await driver.query(systemConnection, dmp.s);
   }
   await handleRefresh();
 }
 
+async function handleDriverDataCore(msgid, callMethod) {
+  await waitConnected();
+  const driver = requireEngineDriver(storedConnection);
+  try {
+    const result = await callMethod(driver);
+    process.send({ msgtype: 'response', msgid, result });
+  } catch (err) {
+    process.send({ msgtype: 'response', msgid, errorMessage: err.message });
+  }
+}
+
+async function handleServerSummary({ msgid }) {
+  return handleDriverDataCore(msgid, driver => driver.serverSummary(systemConnection));
+}
+
+async function handleSummaryCommand({ msgid, command, row }) {
+  return handleDriverDataCore(msgid, driver => driver.summaryCommand(systemConnection, command, row));
+}
+
 const messageHandlers = {
   connect: handleConnect,
   ping: handlePing,
-  createDatabase: handleCreateDatabase,
+  serverSummary: handleServerSummary,
+  summaryCommand: handleSummaryCommand,
+  createDatabase: props => handleDatabaseOp('createDatabase', props),
+  dropDatabase: props => handleDatabaseOp('dropDatabase', props),
 };
 
 async function handleMessage({ msgtype, ...other }) {
@@ -106,11 +146,11 @@ function start() {
 
   setInterval(() => {
     const time = new Date().getTime();
-    if (time - lastPing > 120 * 1000) {
-      console.log('Server connection not alive, exiting');
+    if (time - lastPing > 40 * 1000) {
+      logger.info('Server connection not alive, exiting');
       process.exit(0);
     }
-  }, 60 * 1000);
+  }, 10 * 1000);
 
   process.on('message', async message => {
     if (handleProcessCommunication(message)) return;

packages/api/src/proc/sessionProcess.js

@@ -10,18 +10,29 @@ const requireEngineDriver = require('../utility/requireEngineDriver');
 const { decryptConnection } = require('../utility/crypting');
 const connectUtility = require('../utility/connectUtility');
 const { handleProcessCommunication } = require('../utility/processComm');
+const { getLogger, extractIntSettingsValue, extractBoolSettingsValue } = require('dbgate-tools');
+
+const logger = getLogger('sessionProcess');
 
 let systemConnection;
 let storedConnection;
 let afterConnectCallbacks = [];
 // let currentHandlers = [];
+let lastPing = null;
+let lastActivity = null;
+let currentProfiler = null;
+let executingScripts = 0;
 
 class TableWriter {
-  constructor(structure, resultIndex) {
-    this.jslid = uuidv1();
-    this.currentFile = path.join(jsldir(), `${this.jslid}.jsonl`);
+  constructor() {
     this.currentRowCount = 0;
     this.currentChangeIndex = 1;
+    this.initializedFile = false;
+  }
+
+  initializeFromQuery(structure, resultIndex) {
+    this.jslid = uuidv1();
+    this.currentFile = path.join(jsldir(), `${this.jslid}.jsonl`);
     fs.writeFileSync(
       this.currentFile,
       JSON.stringify({
@@ -32,13 +43,21 @@ class TableWriter {
     this.currentStream = fs.createWriteStream(this.currentFile, { flags: 'a' });
     this.writeCurrentStats(false, false);
     this.resultIndex = resultIndex;
+    this.initializedFile = true;
     process.send({ msgtype: 'recordset', jslid: this.jslid, resultIndex });
   }
 
+  initializeFromReader(jslid) {
+    this.jslid = jslid;
+    this.currentFile = path.join(jsldir(), `${this.jslid}.jsonl`);
+    this.writeCurrentStats(false, false);
+  }
+
   row(row) {
     // console.log('ACCEPT ROW', row);
     this.currentStream.write(JSON.stringify(row) + '\n');
     this.currentRowCount += 1;
+
     if (!this.plannedStats) {
       this.plannedStats = true;
       process.nextTick(() => {
@@ -49,6 +68,21 @@ class TableWriter {
     }
   }
 
+  rowFromReader(row) {
+    if (!this.initializedFile) {
+      process.send({ msgtype: 'initializeFile', jslid: this.jslid });
+      this.initializedFile = true;
+
+      fs.writeFileSync(this.currentFile, JSON.stringify(row) + '\n');
+      this.currentStream = fs.createWriteStream(this.currentFile, { flags: 'a' });
+      this.writeCurrentStats(false, false);
+      this.initializedFile = true;
+      return;
+    }
+
+    this.row(row);
+  }
+
   writeCurrentStats(isFinished = false, emitEvent = false) {
     const stats = {
       rowCount: this.currentRowCount,
@@ -63,18 +97,20 @@ class TableWriter {
     }
   }
 
-  close() {
+  close(afterClose) {
     if (this.currentStream) {
       this.currentStream.end(() => {
         this.writeCurrentStats(true, true);
+        if (afterClose) afterClose();
       });
     }
   }
 }
 
 class StreamHandler {
-  constructor(resultIndexHolder, resolve) {
+  constructor(resultIndexHolder, resolve, startLine) {
     this.recordset = this.recordset.bind(this);
+    this.startLine = startLine;
     this.row = this.row.bind(this);
     // this.error = this.error.bind(this);
     this.done = this.done.bind(this);
@@ -98,7 +134,11 @@ class StreamHandler {
 
   recordset(columns) {
     this.closeCurrentWriter();
-    this.currentWriter = new TableWriter(Array.isArray(columns) ? { columns } : columns, this.resultIndexHolder.value);
+    this.currentWriter = new TableWriter();
+    this.currentWriter.initializeFromQuery(
+      Array.isArray(columns) ? { columns } : columns,
+      this.resultIndexHolder.value
+    );
     this.resultIndexHolder.value += 1;
 
     // this.writeCurrentStats();
@@ -110,7 +150,6 @@ class StreamHandler {
     // }, 500);
   }
   row(row) {
-    // console.log('ACCEPT ROW', row);
     if (this.currentWriter) this.currentWriter.row(row);
     else if (row.message) process.send({ msgtype: 'info', info: { message: row.message } });
     // this.onRow(this.jslid);
@@ -124,22 +163,40 @@ class StreamHandler {
     this.resolve();
   }
   info(info) {
+    if (info && info.line != null) {
+      info = {
+        ...info,
+        line: this.startLine + info.line,
+      };
+    }
     process.send({ msgtype: 'info', info });
   }
 }
 
-function handleStream(driver, resultIndexHolder, sql) {
+function handleStream(driver, resultIndexHolder, sqlItem) {
   return new Promise((resolve, reject) => {
-    const handler = new StreamHandler(resultIndexHolder, resolve);
-    driver.stream(systemConnection, sql, handler);
+    const start = sqlItem.trimStart || sqlItem.start;
+    const handler = new StreamHandler(resultIndexHolder, resolve, start && start.line);
+    driver.stream(systemConnection, sqlItem.text, handler);
   });
 }
 
+function allowExecuteCustomScript(driver) {
+  if (driver.readOnlySessions) {
+    return true;
+  }
+  if (storedConnection.isReadOnly) {
+    return false;
+    // throw new Error('Connection is read only');
+  }
+  return true;
+}
+
 async function handleConnect(connection) {
   storedConnection = connection;
 
   const driver = requireEngineDriver(storedConnection);
-  systemConnection = await connectUtility(driver, storedConnection);
+  systemConnection = await connectUtility(driver, storedConnection, 'app');
   for (const [resolve] of afterConnectCallbacks) {
     resolve();
   }
@@ -159,26 +216,117 @@ function waitConnected() {
   });
 }
 
-async function handleExecuteQuery({ sql }) {
+async function handleStartProfiler({ jslid }) {
+  lastActivity = new Date().getTime();
+
   await waitConnected();
   const driver = requireEngineDriver(storedConnection);
 
-  const resultIndexHolder = {
-    value: 0,
-  };
-  for (const sqlItem of splitQuery(sql, driver.getQuerySplitterOptions('stream'))) {
-    await handleStream(driver, resultIndexHolder, sqlItem);
-    // const handler = new StreamHandler(resultIndex);
-    // const stream = await driver.stream(systemConnection, sqlItem, handler);
-    // handler.stream = stream;
-    // resultIndex = handler.resultIndex;
+  if (!allowExecuteCustomScript(driver)) {
+    process.send({ msgtype: 'done' });
+    return;
   }
-  process.send({ msgtype: 'done' });
+
+  const writer = new TableWriter();
+  writer.initializeFromReader(jslid);
+
+  currentProfiler = await driver.startProfiler(systemConnection, {
+    row: data => writer.rowFromReader(data),
+  });
+  currentProfiler.writer = writer;
+}
+
+async function handleStopProfiler({ jslid }) {
+  lastActivity = new Date().getTime();
+
+  const driver = requireEngineDriver(storedConnection);
+  currentProfiler.writer.close();
+  driver.stopProfiler(systemConnection, currentProfiler);
+  currentProfiler = null;
+}
+
+async function handleExecuteQuery({ sql }) {
+  lastActivity = new Date().getTime();
+
+  await waitConnected();
+  const driver = requireEngineDriver(storedConnection);
+
+  if (!allowExecuteCustomScript(driver)) {
+    process.send({
+      msgtype: 'info',
+      info: {
+        message: 'Connection without read-only sessions is read only',
+        severity: 'error',
+      },
+    });
+    process.send({ msgtype: 'done', skipFinishedMessage: true });
+    return;
+    //process.send({ msgtype: 'error', error: e.message });
+  }
+
+  executingScripts++;
+  try {
+    const resultIndexHolder = {
+      value: 0,
+    };
+    for (const sqlItem of splitQuery(sql, {
+      ...driver.getQuerySplitterOptions('stream'),
+      returnRichInfo: true,
+    })) {
+      await handleStream(driver, resultIndexHolder, sqlItem);
+      // const handler = new StreamHandler(resultIndex);
+      // const stream = await driver.stream(systemConnection, sqlItem, handler);
+      // handler.stream = stream;
+      // resultIndex = handler.resultIndex;
+    }
+    process.send({ msgtype: 'done' });
+  } finally {
+    executingScripts--;
+  }
+}
+
+async function handleExecuteReader({ jslid, sql, fileName }) {
+  lastActivity = new Date().getTime();
+
+  await waitConnected();
+
+  const driver = requireEngineDriver(storedConnection);
+
+  if (fileName) {
+    sql = fs.readFileSync(fileName, 'utf-8');
+  } else {
+    if (!allowExecuteCustomScript(driver)) {
+      process.send({ msgtype: 'done' });
+      return;
+    }
+  }
+
+  const writer = new TableWriter();
+  writer.initializeFromReader(jslid);
+
+  const reader = await driver.readQuery(systemConnection, sql);
+
+  reader.on('data', data => {
+    writer.rowFromReader(data);
+  });
+  reader.on('end', () => {
+    writer.close(() => {
+      process.send({ msgtype: 'done' });
+    });
+  });
+}
+
+function handlePing() {
+  lastPing = new Date().getTime();
 }
 
 const messageHandlers = {
   connect: handleConnect,
   executeQuery: handleExecuteQuery,
+  executeReader: handleExecuteReader,
+  startProfiler: handleStartProfiler,
+  stopProfiler: handleStopProfiler,
+  ping: handlePing,
   // cancel: handleCancel,
 };
 
@@ -189,6 +337,35 @@ async function handleMessage({ msgtype, ...other }) {
 
 function start() {
   childProcessChecker();
+
+  lastPing = new Date().getTime();
+
+  setInterval(() => {
+    const time = new Date().getTime();
+    if (time - lastPing > 25 * 1000) {
+      logger.info('Session not alive, exiting');
+      process.exit(0);
+    }
+
+    const useSessionTimeout =
+      storedConnection && storedConnection.globalSettings
+        ? extractBoolSettingsValue(storedConnection.globalSettings, 'session.autoClose', true)
+        : false;
+    const sessionTimeout =
+      storedConnection && storedConnection.globalSettings
+        ? extractIntSettingsValue(storedConnection.globalSettings, 'session.autoCloseTimeout', 15, 1, 120)
+        : 15;
+    if (
+      useSessionTimeout &&
+      time - lastActivity > sessionTimeout * 60 * 1000 &&
+      !currentProfiler &&
+      executingScripts == 0
+    ) {
+      logger.info('Session not active, exiting');
+      process.exit(0);
+    }
+  }, 10 * 1000);
+
   process.on('message', async message => {
     if (handleProcessCommunication(message)) return;
     try {

packages/api/src/proc/sshForwardProcess.js

@@ -0,0 +1,73 @@
+const fs = require('fs-extra');
+const platformInfo = require('../utility/platformInfo');
+const childProcessChecker = require('../utility/childProcessChecker');
+const { handleProcessCommunication } = require('../utility/processComm');
+const { SSHConnection } = require('../utility/SSHConnection');
+const { getLogger } = require('dbgate-tools');
+
+const logger = getLogger('sshProcess');
+
+async function getSshConnection(connection) {
+  const sshConfig = {
+    endHost: connection.sshHost || '',
+    endPort: connection.sshPort || 22,
+    bastionHost: connection.sshBastionHost || '',
+    agentForward: connection.sshMode == 'agent',
+    passphrase: connection.sshMode == 'keyFile' ? connection.sshKeyfilePassword : undefined,
+    username: connection.sshLogin,
+    password: connection.sshMode == 'userPassword' ? connection.sshPassword : undefined,
+    agentSocket: connection.sshMode == 'agent' ? platformInfo.sshAuthSock : undefined,
+    privateKey:
+      connection.sshMode == 'keyFile' && connection.sshKeyfile ? await fs.readFile(connection.sshKeyfile) : undefined,
+    skipAutoPrivateKey: true,
+    noReadline: true,
+  };
+
+  const sshConn = new SSHConnection(sshConfig);
+  return sshConn;
+}
+
+async function handleStart({ connection, tunnelConfig }) {
+  try {
+    const sshConn = await getSshConnection(connection);
+    await sshConn.forward(tunnelConfig);
+
+    process.send({
+      msgtype: 'connected',
+      connection,
+      tunnelConfig,
+    });
+  } catch (err) {
+    logger.error({ err }, 'Error creating SSH tunnel connection:');
+
+    process.send({
+      msgtype: 'error',
+      connection,
+      tunnelConfig,
+      errorMessage: err.message,
+    });
+  }
+}
+
+const messageHandlers = {
+  connect: handleStart,
+};
+
+async function handleMessage({ msgtype, ...other }) {
+  const handler = messageHandlers[msgtype];
+  await handler(other);
+}
+
+function start() {
+  childProcessChecker();
+  process.on('message', async message => {
+    if (handleProcessCommunication(message)) return;
+    try {
+      await handleMessage(message);
+    } catch (e) {
+      console.error('sshForwardProcess - unhandled error', e);
+    }
+  });
+}
+
+module.exports = { start };

packages/api/src/shell/archiveWriter.js

@@ -3,11 +3,14 @@ const fs = require('fs');
 const { archivedir, resolveArchiveFolder } = require('../utility/directories');
 // const socket = require('../utility/socket');
 const jsonLinesWriter = require('./jsonLinesWriter');
+const { getLogger } = require('dbgate-tools');
+
+const logger = getLogger();
 
 function archiveWriter({ folderName, fileName }) {
   const dir = resolveArchiveFolder(folderName);
   if (!fs.existsSync(dir)) {
-    console.log(`Creating directory ${dir}`);
+    logger.info(`Creating directory ${dir}`);
     fs.mkdirSync(dir);
   }
   const jsonlFile = path.join(dir, `${fileName}.jsonl`);

packages/api/src/shell/copyStream.js

@@ -1,18 +1,47 @@
 const EnsureStreamHeaderStream = require('../utility/EnsureStreamHeaderStream');
+const Stream = require('stream');
+const ColumnMapTransformStream = require('../utility/ColumnMapTransformStream');
+
+function copyStream(input, output, options) {
+  const { columns } = options || {};
+
+  const transforms = [];
+  if (columns) {
+    transforms.push(new ColumnMapTransformStream(columns));
+  }
+  if (output.requireFixedStructure) {
+    transforms.push(new EnsureStreamHeaderStream());
+  }
+
+  // return new Promise((resolve, reject) => {
+  //   Stream.pipeline(input, ...transforms, output, err => {
+  //     if (err) {
+  //       reject(err);
+  //     } else {
+  //       resolve();
+  //     }
+  //   });
+  // });
 
-function copyStream(input, output) {
   return new Promise((resolve, reject) => {
     const finisher = output['finisher'] || output;
     finisher.on('finish', resolve);
     finisher.on('error', reject);
 
-    if (output.requireFixedStructure) {
-      const ensureHeader = new EnsureStreamHeaderStream();
-      input.pipe(ensureHeader);
-      ensureHeader.pipe(output);
-    } else {
-      input.pipe(output);
+    let lastStream = input;
+    for (const tran of transforms) {
+      lastStream.pipe(tran);
+      lastStream = tran;
     }
+    lastStream.pipe(output);
+
+    //  if (output.requireFixedStructure) {
+    //   const ensureHeader = new EnsureStreamHeaderStream();
+    //   input.pipe(ensureHeader);
+    //   ensureHeader.pipe(output);
+    // } else {
+    //   input.pipe(output);
+    // }
   });
 }
 

packages/api/src/shell/dataDuplicator.js

@@ -0,0 +1,50 @@
+const stream = require('stream');
+const path = require('path');
+const { quoteFullName, fullNameToString, getLogger } = require('dbgate-tools');
+const requireEngineDriver = require('../utility/requireEngineDriver');
+const connectUtility = require('../utility/connectUtility');
+const logger = getLogger('dataDuplicator');
+const { DataDuplicator } = require('dbgate-datalib');
+const copyStream = require('./copyStream');
+const jsonLinesReader = require('./jsonLinesReader');
+const { resolveArchiveFolder } = require('../utility/directories');
+
+async function dataDuplicator({
+  connection,
+  archive,
+  items,
+  options,
+  analysedStructure = null,
+  driver,
+  systemConnection,
+}) {
+  if (!driver) driver = requireEngineDriver(connection);
+  const pool = systemConnection || (await connectUtility(driver, connection, 'write'));
+
+  logger.info(`Connected.`);
+
+  if (!analysedStructure) {
+    analysedStructure = await driver.analyseFull(pool);
+  }
+
+  const dupl = new DataDuplicator(
+    pool,
+    driver,
+    analysedStructure,
+    items.map(item => ({
+      name: item.name,
+      operation: item.operation,
+      matchColumns: item.matchColumns,
+      openStream:
+        item.openStream ||
+        (() => jsonLinesReader({ fileName: path.join(resolveArchiveFolder(archive), `${item.name}.jsonl`) })),
+    })),
+    stream,
+    copyStream,
+    options
+  );
+
+  await dupl.run();
+}
+
+module.exports = dataDuplicator;

packages/api/src/shell/dumpDatabase.js

@@ -0,0 +1,41 @@
+const requireEngineDriver = require('../utility/requireEngineDriver');
+const connectUtility = require('../utility/connectUtility');
+const { getLogger } = require('dbgate-tools');
+
+const logger = getLogger('dumpDb');
+
+function doDump(dumper) {
+  return new Promise((resolve, reject) => {
+    dumper.once('end', () => {
+      resolve(true);
+    });
+    dumper.once('error', err => {
+      reject(err);
+    });
+    dumper.run();
+  });
+}
+
+async function dumpDatabase({
+  connection = undefined,
+  systemConnection = undefined,
+  driver = undefined,
+  outputFile,
+  databaseName,
+  schemaName,
+}) {
+  logger.info(`Dumping database`);
+
+  if (!driver) driver = requireEngineDriver(connection);
+  const pool = systemConnection || (await connectUtility(driver, connection, 'read', { forceRowsAsObjects: true }));
+  logger.info(`Connected.`);
+
+  const dumper = await driver.createBackupDumper(pool, {
+    outputFile,
+    databaseName,
+    schemaName,
+  });
+  await doDump(dumper);
+}
+
+module.exports = dumpDatabase;

packages/api/src/shell/executeQuery.js

@@ -1,12 +1,15 @@
 const requireEngineDriver = require('../utility/requireEngineDriver');
 const connectUtility = require('../utility/connectUtility');
+const { getLogger } = require('dbgate-tools');
+
+const logger = getLogger('execQuery');
 
 async function executeQuery({ connection = undefined, systemConnection = undefined, driver = undefined, sql }) {
-  console.log(`Execute query ${sql}`);
+  logger.info({ sql }, `Execute query`);
 
   if (!driver) driver = requireEngineDriver(connection);
-  const pool = systemConnection || (await connectUtility(driver, connection));
-  console.log(`Connected.`);
+  const pool = systemConnection || (await connectUtility(driver, connection, 'script'));
+  logger.info(`Connected.`);
 
   await driver.script(pool, sql);
 }

packages/api/src/shell/fakeObjectReader.js

@@ -1,18 +1,26 @@
 const stream = require('stream');
 
-async function fakeObjectReader({ delay = 0 } = {}) {
+async function fakeObjectReader({ delay = 0, dynamicData = null } = {}) {
   const pass = new stream.PassThrough({
     objectMode: true,
   });
   function doWrite() {
-    pass.write({ columns: [{ columnName: 'id' }, { columnName: 'country' }], __isStreamHeader: true });
-    pass.write({ id: 1, country: 'Czechia' });
-    pass.write({ id: 2, country: 'Austria' });
-    pass.write({ country: 'Germany', id: 3 });
-    pass.write({ country: 'Romania', id: 4 });
-    pass.write({ country: 'Great Britain', id: 5 });
-    pass.write({ country: 'Bosna, Hecegovina', id: 6 });
-    pass.end();
+    if (dynamicData) {
+      pass.write({ __isStreamHeader: true, __isDynamicStructure: true });
+      for (const item of dynamicData) {
+        pass.write(item);
+      }
+      pass.end();
+    } else {
+      pass.write({ columns: [{ columnName: 'id' }, { columnName: 'country' }], __isStreamHeader: true });
+      pass.write({ id: 1, country: 'Czechia' });
+      pass.write({ id: 2, country: 'Austria' });
+      pass.write({ country: 'Germany', id: 3 });
+      pass.write({ country: 'Romania', id: 4 });
+      pass.write({ country: 'Great Britain', id: 5 });
+      pass.write({ country: 'Bosna, Hecegovina', id: 6 });
+      pass.end();
+    }
   }
 
   if (delay) {

packages/api/src/shell/generateDeploySql.js

@@ -21,7 +21,7 @@ async function generateDeploySql({
 }) {
   if (!driver) driver = requireEngineDriver(connection);
 
-  const pool = systemConnection || (await connectUtility(driver, connection));
+  const pool = systemConnection || (await connectUtility(driver, connection, 'read'));
   if (!analysedStructure) {
     analysedStructure = await driver.analyseFull(pool);
   }

packages/api/src/shell/generateModelSql.js

@@ -0,0 +1,30 @@
+const requireEngineDriver = require('../utility/requireEngineDriver');
+const {
+  extendDatabaseInfo,
+  databaseInfoFromYamlModel,
+  getAlterDatabaseScript,
+  DatabaseAnalyser,
+} = require('dbgate-tools');
+const importDbModel = require('../utility/importDbModel');
+const fs = require('fs');
+
+async function generateModelSql({ engine, driver, modelFolder, loadedDbModel, outputFile }) {
+  if (!driver) driver = requireEngineDriver(engine);
+
+  const dbInfo = extendDatabaseInfo(
+    loadedDbModel ? databaseInfoFromYamlModel(loadedDbModel) : await importDbModel(modelFolder)
+  );
+
+  const { sql } = getAlterDatabaseScript(
+    DatabaseAnalyser.createEmptyStructure(),
+    dbInfo,
+    {},
+    DatabaseAnalyser.createEmptyStructure(),
+    dbInfo,
+    driver
+  );
+
+  fs.writeFileSync(outputFile, sql);
+}
+
+module.exports = generateModelSql;

packages/api/src/shell/importDatabase.js

@@ -0,0 +1,60 @@
+const fs = require('fs');
+const requireEngineDriver = require('../utility/requireEngineDriver');
+const connectUtility = require('../utility/connectUtility');
+const { splitQueryStream } = require('dbgate-query-splitter/lib/splitQueryStream');
+const download = require('./download');
+const stream = require('stream');
+const { getLogger } = require('dbgate-tools');
+
+const logger = getLogger('importDb');
+
+class ImportStream extends stream.Transform {
+  constructor(pool, driver) {
+    super({ objectMode: true });
+    this.pool = pool;
+    this.driver = driver;
+  }
+  async _transform(chunk, encoding, cb) {
+    try {
+      await this.driver.script(this.pool, chunk);
+    } catch (err) {
+      this.emit('error', err.message);
+    }
+    cb();
+  }
+  _flush(cb) {
+    this.push('finish');
+    cb();
+    this.emit('end');
+  }
+}
+
+function awaitStreamEnd(stream) {
+  return new Promise((resolve, reject) => {
+    stream.once('end', () => {
+      resolve(true);
+    });
+    stream.once('error', err => {
+      reject(err);
+    });
+  });
+}
+
+async function importDatabase({ connection = undefined, systemConnection = undefined, driver = undefined, inputFile }) {
+  logger.info(`Importing database`);
+
+  if (!driver) driver = requireEngineDriver(connection);
+  const pool = systemConnection || (await connectUtility(driver, connection, 'write'));
+  logger.info(`Connected.`);
+
+  const downloadedFile = await download(inputFile);
+
+  const fileStream = fs.createReadStream(downloadedFile, 'utf-8');
+  const splittedStream = splitQueryStream(fileStream, driver.getQuerySplitterOptions('script'));
+  const importStream = new ImportStream(pool, driver);
+  // @ts-ignore
+  splittedStream.pipe(importStream);
+  await awaitStreamEnd(importStream);
+}
+
+module.exports = importDatabase;

packages/api/src/shell/index.js

@@ -21,6 +21,12 @@ const executeQuery = require('./executeQuery');
 const loadFile = require('./loadFile');
 const deployDb = require('./deployDb');
 const initializeApiEnvironment = require('./initializeApiEnvironment');
+const dumpDatabase = require('./dumpDatabase');
+const importDatabase = require('./importDatabase');
+const loadDatabase = require('./loadDatabase');
+const generateModelSql = require('./generateModelSql');
+const modifyJsonLinesReader = require('./modifyJsonLinesReader');
+const dataDuplicator = require('./dataDuplicator');
 
 const dbgateApi = {
   queryReader,
@@ -45,6 +51,12 @@ const dbgateApi = {
   loadFile,
   deployDb,
   initializeApiEnvironment,
+  dumpDatabase,
+  importDatabase,
+  loadDatabase,
+  generateModelSql,
+  modifyJsonLinesReader,
+  dataDuplicator,
 };
 
 requirePlugin.initializeDbgateApi(dbgateApi);

packages/api/src/shell/jsonArrayWriter.js

@@ -1,6 +1,9 @@
+const { getLogger } = require('dbgate-tools');
 const fs = require('fs');
 const stream = require('stream');
 
+const logger = getLogger('jsonArrayWriter');
+
 class StringifyStream extends stream.Transform {
   constructor() {
     super({ objectMode: true });
@@ -38,7 +41,7 @@ class StringifyStream extends stream.Transform {
 }
 
 async function jsonArrayWriter({ fileName, encoding = 'utf-8' }) {
-  console.log(`Writing file ${fileName}`);
+  logger.info(`Writing file ${fileName}`);
   const stringify = new StringifyStream();
   const fileStream = fs.createWriteStream(fileName, encoding);
   stringify.pipe(fileStream);