#290 removed experimental not working code

Dragwindow

.github/ISSUE_TEMPLATE/bug_report.md

@@ -28,6 +28,3 @@ If applicable, add screenshots to help explain your problem.
  - Install source [e.g. installer/SNAP/Docker/NPM]
  - Type - Web/Application
  - Database engine: [e.g. MySQL/PostgreSQL/SQL Server]
-
-**Additional context**
-Anything else you think might be helpful

.github/ISSUE_TEMPLATE/feature_request.md

@@ -15,6 +15,3 @@ A clear and concise description of what you want to happen.
 
 **Describe alternatives you've considered**
 A clear and concise description of any alternative solutions or features you've considered.
-
-**Additional context**
-Add any other context or screenshots about the feature request here.

.github/workflows/build-app-beta.yaml

@@ -10,22 +10,23 @@ jobs:
     runs-on: ${{ matrix.os }}
 
     strategy:
+      fail-fast: false
       matrix:
-        # os: [ubuntu-18.04, windows-2016]
-        os: [macOS-10.15, windows-2016, ubuntu-18.04]
+        os: [macOS-10.15, windows-2022, ubuntu-18.04]
+        # os: [macOS-10.15]
 
     steps:
       - name: Context
         env:
           GITHUB_CONTEXT: ${{ toJson(github) }}
         run: echo "$GITHUB_CONTEXT"
-      - uses: actions/checkout@v1
+      - uses: actions/checkout@v2
         with:
           fetch-depth: 1
-      - name: Use Node.js 10.x
+      - name: Use Node.js 14.x
         uses: actions/setup-node@v1
         with:
-          node-version: 10.x
+          node-version: 14.x
       - name: yarn install
         run: |
           yarn install
@@ -42,20 +43,19 @@ jobs:
         if: matrix.os == 'ubuntu-18.04'
         uses: samuelmeuli/action-snapcraft@v1
       - name: Publish
-        if: matrix.os != 'macOS-10.15'
         run: |
           yarn run build:app
         env:
           GH_TOKEN: ${{ secrets.GH_TOKEN }} # token for electron publish
+
           WIN_CSC_LINK: ${{ secrets.WINCERT_CERTIFICATE }}
           WIN_CSC_KEY_PASSWORD: ${{ secrets.WINCERT_PASSWORD }}
 
-      - name: Publish Mac
-        if: matrix.os == 'macOS-10.15'
-        run: |
-          yarn run build:app:mac
-        env:
-          GH_TOKEN: ${{ secrets.GH_TOKEN }} # token for electron publish
+          CSC_LINK: ${{ secrets.APPLECERT_CERTIFICATE }}
+          CSC_KEY_PASSWORD: ${{ secrets.APPLECERT_PASSWORD }}
+
+          APPLE_ID: ${{ secrets.APPLE_ID }}
+          APPLE_ID_PASSWORD: ${{ secrets.APPLE_ID_PASSWORD }}
 
       - name: Save snap login
         if: matrix.os == 'ubuntu-18.04'
@@ -78,9 +78,10 @@ jobs:
           cp app/dist/*x86*.AppImage artifacts/dbgate-beta.AppImage || true
           cp app/dist/*arm64*.AppImage artifacts/dbgate-beta-arm64.AppImage || true
           cp app/dist/*armv7l*.AppImage artifacts/dbgate-beta-armv7l.AppImage || true
-          cp app/dist/*.exe artifacts/dbgate-beta.exe || true
-          cp app/dist/*win*.zip artifacts/dbgate-windows-beta.zip || true
-          cp app/dist/*-mac.dmg artifacts/dbgate-beta.dmg || true
+          cp app/dist/*win*.exe artifacts/dbgate-beta.exe || true
+          cp app/dist/*win_x64.zip artifacts/dbgate-windows-beta.zip || true
+          cp app/dist/*win_arm64.zip artifacts/dbgate-windows-beta-arm64.zip || true
+          cp app/dist/*-mac_x64.dmg artifacts/dbgate-beta.dmg || true
           cp app/dist/*-mac_arm64.dmg artifacts/dbgate-beta-arm64.dmg || true
 
           mv app/dist/*.exe artifacts/ || true

.github/workflows/build-app.yaml

@@ -14,24 +14,27 @@ jobs:
     runs-on: ${{ matrix.os }}
 
     strategy:
+      fail-fast: false
       matrix:
         # os: [ubuntu-18.04, windows-2016]
-        os: [macOS-10.15, windows-2016, ubuntu-18.04]
+        os: [macOS-10.15, windows-2022, ubuntu-18.04]
 
     steps:
       - name: Context
         env:
           GITHUB_CONTEXT: ${{ toJson(github) }}
         run: echo "$GITHUB_CONTEXT"
-      - uses: actions/checkout@v1
+      - uses: actions/checkout@v2
         with:
           fetch-depth: 1
-      - name: Use Node.js 10.x
+      - name: Use Node.js 14.x
         uses: actions/setup-node@v1
         with:
-          node-version: 10.x
+          node-version: 14.x
       - name: yarn install
         run: |
+          # yarn --version
+          # yarn config set network-timeout 300000
           yarn install
       - name: setCurrentVersion
         run: |
@@ -50,9 +53,16 @@ jobs:
           yarn run build:app
         env:
           GH_TOKEN: ${{ secrets.GH_TOKEN }} # token for electron publish
+
           WIN_CSC_LINK: ${{ secrets.WINCERT_CERTIFICATE }}
           WIN_CSC_KEY_PASSWORD: ${{ secrets.WINCERT_PASSWORD }}
 
+          CSC_LINK: ${{ secrets.APPLECERT_CERTIFICATE }}
+          CSC_KEY_PASSWORD: ${{ secrets.APPLECERT_PASSWORD }}
+
+          APPLE_ID: ${{ secrets.APPLE_ID }}
+          APPLE_ID_PASSWORD: ${{ secrets.APPLE_ID_PASSWORD }}
+
       - name: generatePadFile
         run: |
           yarn generatePadFile
@@ -79,9 +89,10 @@ jobs:
           cp app/dist/*arm64*.AppImage artifacts/dbgate-latest-arm64.AppImage || true
           cp app/dist/*armv7l*.AppImage artifacts/dbgate-latest-armv7l.AppImage || true
           cp app/dist/*.exe artifacts/dbgate-latest.exe || true
-          cp app/dist/*win*.zip artifacts/dbgate-windows-latest.zip || true
-          cp app/dist/*-mac.dmg artifacts/dbgate-latest.dmg || true
-          cp app/dist/*-mac_arm64.dmg artifacts/dbgate-latest-arm64.dmg || true
+          cp app/dist/*win_x64.zip artifacts/dbgate-windows-latest.zip || true
+          cp app/dist/*win_arm64.zip artifacts/dbgate-windows-latest-arm64.zip || true
+          cp app/dist/*-mac_universal.dmg artifacts/dbgate-latest.dmg || true
+          cp app/dist/*-mac_x64.dmg artifacts/dbgate-latest-x64.dmg || true
 
           mv app/dist/*.exe artifacts/ || true
           mv app/dist/*.zip artifacts/ || true
@@ -115,7 +126,7 @@ jobs:
       #     mv app/dist/latest.yml artifacts/latest.yml || true
 
       - name: Copy latest.yml (windows)
-        if: matrix.os == 'windows-2016'
+        if: matrix.os == 'windows-2022'
         run: |
           mv app/dist/latest.yml artifacts/latest.yml || true
           mv app/dist/dbgate-pad.xml artifacts/ || true

.github/workflows/build-docker-beta.yaml

@@ -6,6 +6,7 @@ on:
   push:
     tags:
       - 'v[0-9]+.[0-9]+.[0-9]+-beta.[0-9]+'
+      - 'v[0-9]+.[0-9]+.[0-9]+-docker.[0-9]+'
 
 jobs:
   build:
@@ -21,15 +22,17 @@ jobs:
         env:
           GITHUB_CONTEXT: ${{ toJson(github) }}
         run: echo "$GITHUB_CONTEXT"
-      - uses: actions/checkout@v1
+      - uses: actions/checkout@v2
         with:
           fetch-depth: 1
-      - name: Use Node.js 10.x
+      - name: Use Node.js 14.x
         uses: actions/setup-node@v1
         with:
-          node-version: 10.x
+          node-version: 14.x
       - name: yarn install
         run: |
+          # yarn --version
+          # yarn config set network-timeout 300000
           yarn install
       - name: setCurrentVersion
         run: |
@@ -45,3 +48,11 @@ jobs:
           docker tag dbgate dbgate/dbgate:beta
           docker login -u ${{ secrets.DOCKER_USERNAME }} -p ${{ secrets.DOCKER_PASSWORD }}
           docker push dbgate/dbgate:beta
+      - name: Build alpine docker image
+        run: |
+          docker build ./docker -t dbgate -f docker/Dockerfile-alpine
+      - name: Push alpine docker image
+        run: |
+          docker tag dbgate dbgate/dbgate:beta-alpine
+          docker login -u ${{ secrets.DOCKER_USERNAME }} -p ${{ secrets.DOCKER_PASSWORD }}
+          docker push dbgate/dbgate:beta-alpine

.github/workflows/build-docker.yaml

@@ -27,13 +27,13 @@ jobs:
         env:
           GITHUB_CONTEXT: ${{ toJson(github) }}
         run: echo "$GITHUB_CONTEXT"
-      - uses: actions/checkout@v1
+      - uses: actions/checkout@v2
         with:
           fetch-depth: 1
-      - name: Use Node.js 10.x
+      - name: Use Node.js 14.x
         uses: actions/setup-node@v1
         with:
-          node-version: 10.x
+          node-version: 14.x
       - name: yarn install
         run: |
           yarn install
@@ -51,3 +51,11 @@ jobs:
           docker tag dbgate dbgate/dbgate
           docker login -u ${{ secrets.DOCKER_USERNAME }} -p ${{ secrets.DOCKER_PASSWORD }}
           docker push dbgate/dbgate
+      - name: Build alpine docker image
+        run: |
+          docker build ./docker -t dbgate -f docker/Dockerfile-alpine
+      - name: Push alpine docker image
+        run: |
+          docker tag dbgate dbgate/dbgate:alpine
+          docker login -u ${{ secrets.DOCKER_USERNAME }} -p ${{ secrets.DOCKER_PASSWORD }}
+          docker push dbgate/dbgate:alpine

.github/workflows/build-npm.yaml

@@ -6,7 +6,7 @@ on:
   push:
     tags:
       - 'v[0-9]+.[0-9]+.[0-9]+'
-      # - 'v[0-9]+.[0-9]+.[0-9]+-alpha.[0-9]+'
+      - 'v[0-9]+.[0-9]+.[0-9]+-alpha.[0-9]+'
 
 # on:
 #   push:
@@ -27,13 +27,13 @@ jobs:
         env:
           GITHUB_CONTEXT: ${{ toJson(github) }}
         run: echo "$GITHUB_CONTEXT"
-      - uses: actions/checkout@v1
+      - uses: actions/checkout@v2
         with:
           fetch-depth: 1
-      - name: Use Node.js 10.x
+      - name: Use Node.js 14.x
         uses: actions/setup-node@v1
         with:
-          node-version: 10.x
+          node-version: 14.x
 
       - name: Configure NPM token
         env:
@@ -79,26 +79,31 @@ jobs:
         run: |
           npm publish
 
-      - name: Publish query-splitter
-        working-directory: packages/query-splitter
-        run: |
-          npm publish
-          
       - name: Publish web
         working-directory: packages/web
         run: |
           npm publish
-                                        
-      - name: Publish dbgate
+
+      - name: Publish dbgate (obsolete)
         working-directory: packages/dbgate
         run: |
           npm publish
 
+      - name: Publish dbgate-serve
+        working-directory: packages/serve
+        run: |
+          npm publish
+
       - name: Publish dbgate-plugin-csv
         working-directory: plugins/dbgate-plugin-csv
         run: |
           npm publish
 
+      - name: Publish dbgate-plugin-xml
+        working-directory: plugins/dbgate-plugin-xml
+        run: |
+          npm publish
+
       - name: Publish dbgate-plugin-excel
         working-directory: plugins/dbgate-plugin-excel
         run: |
@@ -128,3 +133,8 @@ jobs:
         working-directory: plugins/dbgate-plugin-sqlite
         run: |
           npm publish
+
+      - name: Publish dbgate-plugin-redis
+        working-directory: plugins/dbgate-plugin-redis
+        run: |
+          npm publish

.github/workflows/run-tests.yaml

@@ -8,14 +8,14 @@ on:
 jobs:
   test-runner:
     runs-on: ubuntu-latest
-    container: node:10.18-jessie
+    container: node:14.18
 
     steps:
       - name: Context
         env:
           GITHUB_CONTEXT: ${{ toJson(github) }}
         run: echo "$GITHUB_CONTEXT"
-      - uses: actions/checkout@v1
+      - uses: actions/checkout@v2
         with:
           fetch-depth: 1
       - name: yarn install
@@ -31,11 +31,6 @@ jobs:
         run: |
           cd packages/filterparser
           yarn test:ci
-      - name: Query spliiter tests
-        if: always()
-        run: |
-          cd packages/query-splitter
-          yarn test:ci
       - uses: tanmen/jest-reporter@v1
         if: always()
         with:
@@ -48,12 +43,6 @@ jobs:
           github-token: ${{ secrets.GITHUB_TOKEN }}
           result-file: packages/filterparser/result.json
           action-name: Filter parser test results
-      - uses: tanmen/jest-reporter@v1
-        if: always()
-        with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}
-          result-file: packages/query-splitter/result.json
-          action-name: Query splitter test results
 
     services:
       postgres:

CHANGELOG.md

@@ -1,9 +1,310 @@
 # ChangeLog
 
+Builds:
+ - docker - build
+ - npm - npm package dbgate-serve
+ - app - classic electron app
+ - mac - application for macOS
+ - linux - application for linux
+ - win - application for Windows
+
+### 5.0.2
+- FIXED: Cannot use SSH Tunnel after update #291
+
+### 5.0.1
+- FIXED(app): Can't Click Sidebar Menu Item #287
+
+### 5.0.0
+- CHANGED: Connection workflow, connections are opened on tabs instead of modals
+- ADDED: Posibility to connect to DB without saving connection
+- ADDED(mac): Support for SQLite on Mac M1
+- FIXED(mac): Unable to drag window on MacOS #281 #283
+- CHANGED: Renamed dbgate-data directory to .dbgate #248
+- FIXED: Exported SQL has table name undefined #277
+- ADDED: More data types in table create dialogt #285
+- ADDED(app): Open previously saved ERD diagrams #278
+- CHANGED: Better app loading progress UX
+- FIXED: Removed SSL tab on Redis connection (SSL is not supported for Redis)
+
+### 4.8.8
+- CHANGED: New app icon
+- ADDED: SQL dump, SQL import - also from/to saved queries
+- FIXED(mac): Fixed crash when reopening main window
+- FIXED: MySQL dump now handles correctly dependand views
+- FIXED(app): Browse tabs with Ctrl+Tab
+- ADDED(app): Browse tabs in reverse order with Ctrl+Shift+Tab #245
+
+### 4.8.7
+- ADDED: MySQL dump/backup database
+- ADDED: Import SQL dump from file or from URL
+- FIXED(mac): Fixed Cmd+C, Cmd+V, Cmd+X - shortcuts for copy/cut/paste #270
+- FIXED(mac): Some minor issues on macOS
+- FIXED: Analysing MS SQL nvarchar(max)
+- ADDED: Support for dockerhost network name under docker #271
+
+### 4.8.4
+- FIXED(mac): Fixed build for macOS arm64 #259
+- FIXED(mac): Fixed opening SQLite files on macOS #243
+- FIXED(mac): Fixed opening PEM certificates on macOS #206
+- FIXED(mac): Fixed handling Command key on macOS
+- FIXED(mac): Fixed system menu on macOS
+- FIXED(mac): Fixed reopening main window on macOS
+- CHANGED: Shortcut for net query is now Ctrl+T or Command+T on macOS, former it was Ctrl+Q
+- FIXED: Fixed misplaced tab close icon #260
+- ADDED: Added menu command "Tools/Change to recent database"
+
+### 4.8.3
+- FIXED: filters in query result and NDJSON/archive viewer
+- ADDED: Added select values from query result and NDJSON/archive viewer
+- ADDED: tab navigation in datagrid #254
+- ADDED: Keyboard shortcuts added to help menu #254
+- ADDED: API logging (run enableApiLog() in developers console to enable logging)
+- ADDED: SSH reconnect + moved SSH forward into separate fork #253
+- ADDED: Data type + reference link in column manager
+- FIXED(win,linux,mac): Unable to change theme after installing plugin #244
+
+ ### 4.8.2
+ - ADDED: implemented missing redis search key logic
+
+ ### 4.8.1
+ - FIXED: fixed crash after disconnecting from all DBs
+
+### 4.8.0
+- ADDED: Redis support (support stream type), removed experimental status
+- ADDED: Redis readonly support
+- ADDED: Explicit NDJSON support, when opening NDJSON/JSON lines file, table data are immediately shown, without neccesarity to import
+- ADDED(win,linux,mac): Opening developer tools when crashing without reload app
+### 4.7.4
+- ADDED: Experimental Redis support (full support is planned to version  4.8.0)
+- ADDED: Read-only connections
+- FIXED: MongoDB filters
+- ADDED: MongoDB column value selection
+- ADDED: App related queries
+- ADDED: Fuzzy search #246
+- ADDED(docker, npm): New permissions
+- FIXED(npm): NPM build no longer allocates additonal ports
+- CHANGED(npm): renamed NPM package dbgate => dbgate-serve 
+- CHANGED(docker): custom JavaScripts and connections defined in scripts are now prohibited by default, use SHELL_CONNECTION and SHELL_SCRIPTING environment variables for allowing this
+- ADDED(docker, npm): Better documentation of environment variables configuration, https://dbgate.org/docs/env-variables.html
+- ADDED(docker): support for multiple users with different permissions
+- ADDED(docker): logout operation
+
+### 4.7.3
+- CHANGED: Export menu redesign, quick export menu merged with old export menu
+- REMOVED: Quick export menu
+- ADDED: Export column mapping
+- ADDED: Export invoked from data grid respects columns choosed in column manager
+- ADDED: Quick export (now merged in export menu) is now possible also in web app
+- FIXED: Virtual foreign key editor fixes
+- FIXED: Tabs panel style fix
+- ADDED: Find by schema in databases widget
+- FIXED: Column manager selection fix
+- FIXED: NPM dist - fixed error when loading plugins
+- CHANGED: NPN dist is now executed by dbgate-serve command
+- ADDED: NPM dist accepts .env configuration
+
+### 4.7.2
+- CHANGED: documentation URL - https://dbgate.org/docs/
+- CHANGED: Close button available for all tab groups - #238
+- ADDED: Search function for the Keyboard Shortcuts overview - #239
+- ADDED: Editor font size settings - #229
+- ADDED: Rename MongoDB collection - #223
+- FIXED: bug in cache subsystem
+
+### 4.7.1
+- FIXED: Fixed connecting to MS SQL server running in docker container from DbGate running in docker container #236 
+- FIXED: Fixed export MongoDB collections into Excel and CSV #240
+- ADDED: Added support for docker volumes to persiste connections, when not using configuration via env variables #232
+- ADDED: DbGate in Docker can run in subdirectory #228
+- FIXED: DbGate in Docker can be proxied with nginx #228
+- FIDED: Theme persists when opening multiple windows #207
+- ADDED: Remember fullscreen state #230
+- ADDED: Improved fullscreen state, title bar with menu is hidden, menu is in hamburger menu, like in web version
+- ADDED: Theme choose dialog (added as tab in settings)
+- FIXED: Fixed crash when clicking on application layers #231
+### 4.7.0
+- CHANGED: Changed main menu style, menu and title bar is in one line (+ability to switch to system menu)
+- REMOVED: Removed main toolbar, use main menu or tab related bottom tool instead
+- ADDED: Added tab related context bottom toolbar
+- ADDED: Main menu is available also in web application, by clicking on hamburger menu
+- ADDED: Added support of SQLite to docker container #219
+- ADDED: Added Debian and Alpine docker distributions (default is Debian)
+- FIXED: Fixed performance problem of data grid, especially when there are cells with large data (eg. JSONs), now it is much faster
+- ADDED: Open JSON and array cell buttons
+- ADDED: Handle JSON in varchar cells
+- ADDED: Scroll tabs on mouse wheel
+- ADDED: Show edit edit MySQL column comments #218 #81
+- ADDED: Handle sparse (mssql), unsigned (mysql), zerofill (mysql) column flags
+- FIXED: Fixed same caching problems (eg. leading to indefinitely loading DB structure sometimes)
+- ADDED: Show estimated table row count for MySQL and MS SQL
+- FIXED: Fixed deleting rows from added rows in table data editor
+- ADDED: Better work with JSON lines file, added JSONL editor with preview
+
+### 4.6.3
+- FIXED: Fixed Windows build
+- FIXED: Fixed crash, when there is invalid value in browser local storage
+- FIXED: Fixed plugin description display, where author name or description is not correctly filled
+
+### 4.6.2
+- FIXED: Fixed issues of XML import plugin
+- ADDED: Split columns macro (available in data sheet editor)
+- CHANGED: Accepting non standard plugins names (which doesn't start with dbgate-plugin-)
+- ADDED: Support BLOB values #211
+- ADDED: Picture cell view
+- ADDED: HTML cell view
+- CHANGED: Code completion supports non-default schema names
+- FIXED: More robust MySQL analyser, when connecting to non-standard servers #214
+- FIXED: Fixed configuring connection to SQLite with environment variables #215
+
+### 4.6.1
+- ADDED: Ability to configure SSH tunnel over environment variables #210 (for docker container)
+- ADDED: XML export and import
+- ADDED: Archive file - show and edit source text file
+- ADDED: Window title shows current tab and database
+- ADDED: DbGate documentation
+- ADDED: Introduced application layers
+- ADDED: Virtual foreign key editor
+- ADDED: Application commands (SQL scripts related to database)
+- ADDED: Theme can be implemented in plugin
+- CHANGED: Dictionary description is stored in app
+- FIXED: Unique and index editor
+- FIXED: Posibility to edit UNIQUE index flag
+- CHANGED: UX improvements of table editor
+
+### 4.6.0
+- ADDED: ER diagrams #118
+    - Generate diagram from table or for database
+    - Automatic layout
+    - Diagram styles - colors, select columns to display, optional displaying data type or nullability
+    - Export diagram to HTML file
+- FIXED: Mac latest build link #204
+
+### 4.5.1
+- FIXED: MongoId detection
+- FIXED: #203 disabled spellchecker
+- FIXED: Prevented display filters in form view twice
+- FIXED: Query designer fixes
+
+### 4.5.0
+- ADDED: #220 functions, materialized views and stored procedures in code completion
+- ADDED: Query result in statusbar
+- ADDED: Highlight and execute current query
+- CHANGED: Code completion offers objects only from current query
+- CHANGED: Big optimalizations of electron app - removed embedded web server, removed remote module, updated electron to version 13
+- CHANGED: Removed dependency to electron-store module
+- FIXED: #201 fixed database URL definition, when running from Docvker container
+- FIXED: #192 Docker container stops in 1 second, ability to stop container with Ctrl+C
+- CHANGED: Web app - websocket replaced with SSE technology
+- CHANGED: Changed tab order, tabs are ordered by creation time
+- ADDED: Reorder tabs with drag & drop
+- CHANGED: Collapse left column in datagrid - removed from settings, remember last used state
+- ADDED: Ability to select multiple columns in column manager in datagrid + copy column names
+- ADDED: Show used filters in left datagrid column
+- FIXED: Fixed delete dependency cycle detection (delete didn't work for some tables)
+
+### 4.4.4
+- FIXED: Database colors
+- CHANGED: Precise work with MongoDB ObjectId
+- FIXED: Run macro works on MongoDB collection data editor
+- ADDED: Type conversion macros
+- CHANGED: Improved UX of import into current database or current archive
+- ADDED: Posibility to create string MongoDB IDs when importing into MongoDB collections
+- CHANGED: Better crash recovery
+- FIXED: Context menu of data editor when using views - some commands didn't work for views
+- ADDED: Widget lists (on left side) now supports add operation, where it has sense
+- CHANGED: Improved UX of saved data sheets
+- ADDED: deploy - preloadedRows: impelemnted onsertOnly columns
+- ADDED: Show change log after app upgrade
+
+### 4.4.3
+- ADDED: Connection and database colors
+- ADDED: Ability to pin connection or table
+- ADDED: MongoDb: create, drop collection from menu
+- ADDED: Copy as MongoDB insert
+- ADDED: MongoDB support for multiple statements in script (dbgate-query-splitter)
+- ADDED: View JSON in tab
+- ADDED: Open DB model as JSON
+- ADDED: Open JSON array as data sheet
+- ADDED: Open JSON from data grid
+- FIXED: Mongo update command when using string IDs resembling Mongo IDs
+- CHANGED: Imrpoved add JSON document, change JSON document commands
+- ADDED: Possibility to add column to JSON grid view
+- FIXED: Hiding columns #1
+- REMOVED: Copy JSON document menu command (please use Copy advanced instead)
+- CHANGED: Save widget visibility and size
+
+### 4.4.2
+- ADDED: Open SQL script from SQL confirm
+- CHANGED: Better looking statusbar
+- ADDED: Create table from database popup menu
+- FIXED: Some fixes for DB compare+deploy (eg. #196)
+- ADDED: Archives + DB models from external directories
+- ADDED: DB deploy supports preloaded data
+- ADDED: Support for Command key on Mac (#199)
+
+### 4.4.1
+- FIXED: #188 Fixed problem with datetime values in PostgreSQL and mysql
+- ADDED: #194 Close tabs by DB
+- FIXED: Improved form view width calculations
+- CHANGED: Form view - highlight matched columns instead of filtering
+- ADDED: Lookup distinct values
+- ADDED: Copy advanced command, Copy as CSV, JSON, YAML, SQL
+- CHANGED: Hide column manager by default
+- ADDED: Change database status command
+- CHANGED: Table structure and view structure tabs have different icons
+- ADDED: #186 - zoom setting
+- ADDED: Row count information moved into status bar, when only one grid on tab is used (typical case)
+
+### 4.4.0
+- ADDED: Database structure compare, export report to HTML
+- ADDED: Experimental: Deploy DB structure changes between databases
+- ADDED: Lookup dialog, available in table view on columns with foreign key
+- ADDED: Customize foreign key lookups
+- ADDED: Chart improvements, export charts as HTML page
+- ADDED: Experimental: work with DB model, deploy model, compare model with real DB
+- ADDED: #193 new SQLite db command
+- CHANGED: #190 code completion improvements
+- ADDED: #189 Copy JSON document - context menu command in data grid for MongoDB
+- ADDED: #191 Connection to POstgreSQL can be defined also with connection string
+- ADDED: #187 dbgate-query-splitter: Transform stream support
+- CHANGED: Upgraded to node 12 in docker app
+- FIXED: Upgraded to node 12 in docker app
+- FIXED: Fixed import into SQLite and PostgreSQL databases, added integration test for this
+
+### 4.3.4
+- FIXED: Delete row with binary ID in MySQL (#182)
+- ADDED: Using 'ODBC Driver 17 for SQL Server' or 'SQL Server Native Client 11.0', when connecting to MS SQL using windows auth #183
+
+### 4.3.3
+- ADDED: Generate SQL from data (#176 - Copy row as INSERT/UPDATE statement)
+- ADDED: Datagrid keyboard column operations (Ctrl+F - find column, Ctrl+H - hide column) #180
+- FIXED: Make window remember that it was maximized
+- FIXED: Fixed lost focus after copy to clipboard and after inserting SQL join
+
+### 4.3.2
+- FIXED: Sorted database list in PostgreSQL (#178)
+- FIXED: Loading stricture of PostgreSQL database, when it contains indexes on expressions (#175)
+- ADDED: Hotkey Shift+Alt+F for formatting SQL code
+
+### 4.3.1
+- FIXED: #173 Using key phrase for SSH key file connection
+- ADDED: #172 Abiloity to quick search within database names
+- ADDED: Database search added to command palette (Ctrl+P)
+- FIXED: #171 fixed PostgreSQL analyser for older versions than 9.3 (matviews don't exist)
+- ADDED: DELETE cascade option - ability to delete all referenced rows, when deleting rows
+
+### 4.3.0
+- ADDED: Table structure editor
+- ADDED: Index support
+- ADDED: Unique constraint support
+- ADDED: Context menu for drop/rename table/columns and for drop view/procedure/function
+- ADDED: Added support for Windows arm64 platform
+- FIXED: Search by _id in MongoDB
+
 ### 4.2.6
- - Fixed MongoDB import
- - Configurable thousands separator #136
- - Using case insensitive text search in postgres
+- FIXED: Fixed MongoDB import
+- ADDED: Configurable thousands separator #136
+- ADDED: Using case insensitive text search in postgres
 
 ### 4.2.5
 - FIXED: Fixed crash when using large model on some installations

README.md

@@ -1,33 +1,59 @@
-[![NPM version](https://img.shields.io/npm/v/dbgate.svg)](https://www.npmjs.com/package/dbgate)
+[![NPM version](https://img.shields.io/npm/v/dbgate-serve.svg)](https://www.npmjs.com/package/dbgate-serve)
 ![GitHub All Releases](https://img.shields.io/github/downloads/dbgate/dbgate/total) 
 [![dbgate](https://snapcraft.io/dbgate/badge.svg)](https://snapcraft.io/dbgate)
 [![dbgate](https://snapcraft.io/dbgate/trending.svg?name=0)](https://snapcraft.io/dbgate)
 [![styled with prettier](https://img.shields.io/badge/styled_with-prettier-ff69b4.svg)](https://github.com/prettier/prettier)
 
-# DbGate - database administration tool
+<img src="https://raw.githubusercontent.com/dbgate/dbgate/master/app/icon.png" width="64" align="right"/>
 
-DbGate modern, fast and easy to use database manager
+# DbGate - (no)SQL database client
+
+DbGate is cross-platform database manager. 
+It's designed to be simple to use and effective, when working with more databases simultaneously.
+But there are also many advanced features like schema compare, visual query designer, chart visualisation or batch export and import.
+
+DbGate is licensed under MIT license and is completely free.
 
 * Try it online - [demo.dbgate.org](https://demo.dbgate.org) - online demo application
-* Download application for Windows, Linux or Mac from [dbgate.org](https://dbgate.org/download/)
-* Run web version as [NPM package](https://www.npmjs.com/package/dbgate) or as [docker image](https://hub.docker.com/r/dbgate/dbgate)
+* **Download** application for Windows, Linux or Mac from [dbgate.org](https://dbgate.org/download/)
+* Run web version as [NPM package](https://www.npmjs.com/package/dbgate-serve) or as [docker image](https://hub.docker.com/r/dbgate/dbgate)
 
-Supported databases:
+## Supported databases
 * MySQL
 * PostgreSQL
 * SQL Server
 * MongoDB
+* Redis
 * SQLite
 * Amazon Redshift
 * CockroachDB
 * MariaDB
 
-![Screenshot](https://raw.githubusercontent.com/dbgate/dbgate/master/screenshot.png)
+<!-- Learn more about DbGate features at the [DbGate website](https://dbgate.org/), or try our online [demo application](https://demo.dbgate.org) -->
+
+
+<a href="https://raw.githubusercontent.com/dbgate/dbgate/master/img/screenshot1.png">
+    <img src="https://raw.githubusercontent.com/dbgate/dbgate/master/img/screenshot1.png" width="400"/>
+</a>
+<a href="https://raw.githubusercontent.com/dbgate/dbgate/master/img/screenshot2.png">
+    <img src="https://raw.githubusercontent.com/dbgate/dbgate/master/img/screenshot2.png" width="400"/>
+</a>
+<a href="https://raw.githubusercontent.com/dbgate/dbgate/master/img/screenshot4.png">
+    <img src="https://raw.githubusercontent.com/dbgate/dbgate/master/img/screenshot4.png" width="400"/>
+</a>
+<a href="https://raw.githubusercontent.com/dbgate/dbgate/master/img/screenshot3.png">
+    <img src="https://raw.githubusercontent.com/dbgate/dbgate/master/img/screenshot3.png" width="400"/>
+</a>
+
+<!-- ![Screenshot](https://raw.githubusercontent.com/dbgate/dbgate/master/screenshot.png) -->
 
 ## Features
 * Table data editing, with SQL change script preview
+* Edit table schema, indexes, primary and foreign keys
+* Compare and synchronize database structure
+* ER diagram
 * Light and dark theme
-* Master/detail views
+* Master/detail views, foreign key lookups
 * Query designer
 * Form view for comfortable work with tables with many columns
 * JSON view on MongoDB collections
@@ -38,35 +64,42 @@ Supported databases:
   * SQL code completion
   * Add SQL LEFT/INNER/RIGHT join utility
 * Mongo JavaScript editor, execute Mongo script (with NodeJs syntax)
+* Redis tree view, generate script from keys, run Redis script
 * Runs as application for Windows, Linux and Mac. Or in Docker container on server and in web Browser on client.
-* Import, export from/to CSV, Excel, JSON
+* Import, export from/to CSV, Excel, JSON, XML
 * Free table editor - quick table data editing (cleanup data after import/before export, prototype tables etc.)
 * Archives - backup your data in JSON files on local filesystem (or on DbGate server, when using web application)
-* Charts
+* Charts, export chart to HTML page
 * For detailed info, how to run DbGate in docker container, visit [docker hub](https://hub.docker.com/r/dbgate/dbgate)
 * Extensible plugin architecture
 
 ## How to contribute
 Any contributions are welcome. If you want to contribute without coding, consider following:
-* Create issue, if you find problem in app, or you have idea to new feature. If issue already exists, you could leave comment on it, to prioritise most wanted issues.
+
 * Tell your friends about DbGate or share on social networks - when more people will use DbGate, it will grow to be better
+* Write review on [Slant.co](https://www.slant.co/improve/options/41086/~dbgate-review) or [G2](https://www.g2.com/products/dbgate/reviews) 
+* Create issue, if you find problem in app, or you have idea to new feature. If issue already exists, you could leave comment on it, to prioritise most wanted issues.
+* Become a backer on [Open collective](https://opencollective.com/dbgate)
+* Where a small coding is acceptable for you, you could [create plugin](https://dbgate.org/docs/plugin-development.html). Plugins for new themes can be created actually without JS coding.
+
+Thank you!
 
 ## Why is DbGate different
 There are many database managers now, so why DbGate? 
 * Works everywhere - Windows, Linux, Mac, Web browser (+mobile web is planned), without compromises in features
 * Based on standalone NPM packages, scripts can be run without DbGate (example - [CSV export](https://www.npmjs.com/package/dbgate-plugin-csv) )
-* Many data browsing functions based using foreign keys - master/detail, expand columns, expandable form view (on screenshot above)
+* Many data browsing functions based using foreign keys - master/detail, expand columns, expandable form view
 
 ## Design goals
 * Application simplicity - DbGate takes the best and only the best from old [DbGate](http://www.jenasoft.com/dbgate), [DatAdmin](http://www.jenasoft.com/datadmin) and [DbMouse](http://www.jenasoft.com/dbmouse) .
 * Minimal dependencies
-    * Frontend - Svelte, socket.io
-    * Backend - NodeJs, ExpressJs, socket.io, database connection drivers
+    * Frontend - Svelte
+    * Backend - NodeJs, ExpressJs, database connection drivers
     * JavaScript + TypeScript
     * App - electron
-* Platform independent - will run as web application in single docker container on server, or as application using Electron platform on Linux, Windows and Mac
+* Platform independent - runs as web application in single docker container on server, or as application using Electron platform on Linux, Windows and Mac
 
-## Plugins
+<!-- ## Plugins
 Plugins are standard NPM packages published on [npmjs.com](https://www.npmjs.com).  
 See all [existing DbGate plugins](https://www.npmjs.com/search?q=keywords:dbgateplugin).  
 Visit [dbgate generator homepage](https://github.com/dbgate/generator-dbgate) to see, how to create your own plugin.  
@@ -75,31 +108,46 @@ Currently following extensions can be implemented using plugins:
 - File format parsers/writers
 - Database engine connectors
 
-Basic set of plugins is part of DbGate git repository and is installed with app. Additional plugins pust be downloaded from NPM (this task is handled by DbGate)
+Basic set of plugins is part of DbGate git repository and is installed with app. Additional plugins pust be downloaded from NPM (this task is handled by DbGate) -->
 
 ## How to run development environment
 
+Simple variant - runs WEB application:
 ```sh
 yarn
 yarn start
 ```
 
-If you want to make modifications in libraries or plugins, run library compiler in watch mode in the second terminal:
+If you want more control, run WEB application:
 ```sh
-yarn lib
+yarn # install NPM packages
 ```
 
-Open http://localhost:5000 in your browser
+And than run following 3 commands concurrently in 3 terminals:
+```
+yarn start:api # run API on port 3000
+yarn start:web # run web on port 5001
+yarn lib # watch typescript libraries and plugins modifications
+```
+This runs API on port 3000 and web application on port 5001  
+Open http://localhost:5001 in your browser
 
-You could run electron app (requires running localhost:5000):
+If you want to run electron app:
 ```sh
+yarn # install NPM packages
 cd app
-yarn
-yarn start
+yarn # install NPM packages for electron
+```
+
+And than run following 3 commands concurrently in 3 terminals:
+```
+yarn start:web # run web on port 5001 (only static JS and HTML files)
+yarn lib # watch typescript libraries and plugins modifications
+yarn start:app # run electron app
 ```
 
 ## How to run built electron app locally
-This mode is very similar to production run of electron app. Electron app forks process with API on dynamically allocated port, works with compiled javascript files and uses compiled version of plugins (doesn't use localhost:5000)
+This mode is very similar to production run of electron app. Electron doesn't use localhost:5001.
 
 ```sh
 cd app
@@ -112,3 +160,16 @@ yarn build:app:local
 yarn start:app:local
 ```
 
+## How to create plugin
+Creating plugin is described in [documentation](https://github.com/dbgate/dbgate/wiki/Plugin-development)
+
+But it is very simple:
+
+```sh
+npm install -g yo # install yeoman
+npm install -g generator-dbgate # install dbgate generator
+cd dbgate-plugin-my-new-plugin # this directory is created by wizard, edit, what you need to change
+yarn plugin # this compiles plugin and copies it into existing DbGate installation
+```
+
+After restarting DbGate, you could use your new plugin from DbGate.

app/.npmrc

@@ -0,0 +1 @@
+better-sqlite3_local_prebuilds=../../prebuilds

app/entitlements.mac.plist

@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+  <dict>
+    <key>com.apple.security.cs.disable-library-validation</key><true/>
+    <key>com.apple.security.cs.allow-jit</key><true/>
+    <key>com.apple.security.cs.allow-unsigned-executable-memory</key><true/>
+    <key>com.apple.security.cs.allow-dyld-environment-variables</key><true/>
+    <key>com.apple.security.files.user-selected.read-only</key><true/>
+    <key>com.apple.security.files.user-selected.read-write</key><true/>
+  </dict>
+</plist>

app/package.json

@@ -1,14 +1,13 @@
 {
   "name": "dbgate",
-  "version": "4.1.1",
+  "version": "5.0.0-alpha.1",
   "private": true,
   "author": "Jan Prochazka <jenasoft.database@gmail.com>",
   "description": "Opensource database administration tool",
   "dependencies": {
-    "better-sqlite3-with-prebuilds": "^7.1.8",
-    "electron-log": "^4.3.1",
-    "electron-store": "^5.1.1",
-    "electron-updater": "^4.3.5",
+    "electron-log": "^4.4.1",
+    "electron-updater": "^4.6.1",
+    "lodash.clonedeepwith": "^4.5.0",
     "patch-package": "^6.4.7"
   },
   "repository": {
@@ -16,18 +15,23 @@
     "url": "https://github.com/dbgate/dbgate.git"
   },
   "build": {
-    "artifactName": "${productName}-${version}-${os}_${arch}.${ext}",
+    "artifactName": "dbgate-${version}-${os}_${arch}.${ext}",
     "appId": "org.dbgate",
+    "productName": "DbGate",
+    "afterSign": "electron-builder-notarize",
     "mac": {
       "category": "database",
       "icon": "icon512.png",
+      "hardenedRuntime": true,
+      "entitlements": "entitlements.mac.plist",
+      "entitlementsInherit": "entitlements.mac.plist",
       "publish": [
         "github"
       ],
       "target": {
         "target": "default",
         "arch": [
-          "arm64",
+          "universal",
           "x64"
         ]
       }
@@ -68,7 +72,13 @@
     "win": {
       "target": [
         "nsis",
-        "zip"
+        {
+          "target": "zip",
+          "arch": [
+            "x64",
+            "arm64"
+          ]
+        }
       ],
       "icon": "icon.ico",
       "publish": [
@@ -84,23 +94,25 @@
   },
   "homepage": "./",
   "scripts": {
-    "start": "cross-env ELECTRON_START_URL=http://localhost:5000 electron .",
+    "start": "cross-env ELECTRON_START_URL=http://localhost:5001 DEVMODE=1 electron .",
     "start:local": "cross-env electron .",
     "dist": "electron-builder",
     "build": "cd ../packages/api && yarn build && cd ../web && yarn build && cd ../../app && yarn dist",
-    "build:mac": "cd ../packages/api && yarn build && cd ../web && yarn build && cd ../../app && node setMacPlatform x64 && yarn dist && node setMacPlatform arm64 && yarn dist",
     "build:local": "cd ../packages/api && yarn build && cd ../web && yarn build && cd ../../app && yarn predist",
-    "postinstall": "electron-builder install-app-deps && patch-package",
+    "postinstall": "yarn rebuild && patch-package",
+    "rebuild": "electron-builder install-app-deps",
     "predist": "copyfiles ../packages/api/dist/* packages && copyfiles \"../packages/web/public/*\" packages && copyfiles \"../packages/web/public/**/*\" packages && copyfiles --up 3 \"../plugins/dist/**/*\" packages/plugins"
   },
   "main": "src/electron.js",
   "devDependencies": {
     "copyfiles": "^2.2.0",
     "cross-env": "^6.0.3",
-    "electron": "11.2.3",
-    "electron-builder": "22.10.5"
+    "electron": "13.6.3",
+    "electron-builder": "22.14.5",
+    "electron-builder-notarize": "^1.4.0"
   },
   "optionalDependencies": {
-    "msnodesqlv8": "^2.0.10"
+    "better-sqlite3": "7.5.0",
+    "msnodesqlv8": "^2.4.4"
   }
 }

app/setMacPlatform.js

@@ -1,8 +0,0 @@
-const fs = require('fs');
-
-const text = fs.readFileSync('package.json', { encoding: 'utf-8' });
-const json = JSON.parse(text);
-
-json.build.mac.target.arch = process.argv[2];
-
-fs.writeFileSync('package.json', JSON.stringify(json, null, 2), { encoding: 'utf-8' });

app/src/electron.js

@@ -1,10 +1,10 @@
 const electron = require('electron');
 const os = require('os');
+const fs = require('fs');
 const { Menu, ipcMain } = require('electron');
-const { fork } = require('child_process');
 const { autoUpdater } = require('electron-updater');
-const Store = require('electron-store');
 const log = require('electron-log');
+const _cloneDeepWith = require('lodash.clonedeepwith');
 
 // Module to control application life.
 const app = electron.app;
@@ -13,13 +13,31 @@ const BrowserWindow = electron.BrowserWindow;
 
 const path = require('path');
 const url = require('url');
+const mainMenuDefinition = require('./mainMenuDefinition');
+const { settings } = require('cluster');
 
-const store = new Store();
+// require('@electron/remote/main').initialize();
+
+const configRootPath = path.join(app.getPath('userData'), 'config-root.json');
+let initialConfig = {};
+let apiLoaded = false;
+let mainModule;
+let winCounter = 0;
+
+const isMac = () => os.platform() == 'darwin';
+
+try {
+  initialConfig = JSON.parse(fs.readFileSync(configRootPath, { encoding: 'utf-8' }));
+} catch (err) {
+  console.log('Error loading config-root:', err.message);
+  initialConfig = {};
+}
 
 // Keep a global reference of the window object, if you don't, the window will
 // be closed automatically when the JavaScript object is garbage collected.
-let mainWindow;
-let mainMenu;
+const mainWindows = {};
+const mainMenus = {};
+let runCommandOnLoad = null;
 
 log.transports.file.level = 'debug';
 autoUpdater.logger = log;
@@ -28,94 +46,71 @@ autoUpdater.logger = log;
 
 let commands = {};
 
-function commandItem(id) {
+function formatKeyText(keyText) {
+  if (!keyText) {
+    return keyText;
+  }
+  if (os.platform() == 'darwin') {
+    return keyText.replace('CtrlOrCommand+', 'Command+');
+  }
+  return keyText.replace('CtrlOrCommand+', 'Ctrl+');
+}
+
+function commandItem(item) {
+  const id = item.command;
   const command = commands[id];
+  if (item.skipInApp) {
+    return { skip: true };
+  }
   return {
     id,
     label: command ? command.menuName || command.toolbarName || command.name : id,
-    accelerator: command ? command.keyText : undefined,
+    accelerator: formatKeyText(command ? command.keyText : undefined),
     enabled: command ? command.enabled : false,
     click() {
-      mainWindow.webContents.executeJavaScript(`dbgate_runCommand('${id}')`);
+      if (mainWindow) {
+        mainWindow.webContents.send('run-command', id);
+      } else {
+        runCommandOnLoad = id;
+        createWindow();
+      }
     },
   };
 }
 
 function buildMenu() {
-  const template = [
-    {
-      label: 'File',
-      submenu: [
-        commandItem('new.connection'),
-        commandItem('file.open'),
-        commandItem('group.save'),
-        commandItem('group.saveAs'),
-        { type: 'separator' },
-        { role: 'close' },
-      ],
-    },
-    {
-      label: 'Window',
-      submenu: [commandItem('new.query'), { type: 'separator' }, commandItem('tabs.closeAll'), { role: 'minimize' }],
-    },
+  let template = _cloneDeepWith(mainMenuDefinition({ editMenu: true }), item => {
+    if (item.divider) {
+      return { type: 'separator' };
+    }
 
-    // {
-    //   label: 'Edit',
-    //   submenu: [
-    //     { role: 'undo' },
-    //     { role: 'redo' },
-    //     { type: 'separator' },
-    //     { role: 'cut' },
-    //     { role: 'copy' },
-    //     { role: 'paste' },
-    //   ],
-    // },
-    {
-      label: 'View',
-      submenu: [
-        { role: 'reload' },
-        { role: 'forcereload' },
-        { role: 'toggledevtools' },
-        { type: 'separator' },
-        { role: 'resetzoom' },
-        { role: 'zoomin' },
-        { role: 'zoomout' },
-        { type: 'separator' },
-        { role: 'togglefullscreen' },
-        commandItem('theme.changeTheme'),
-      ],
-    },
-    {
-      role: 'help',
-      submenu: [
-        {
-          label: 'dbgate.org',
-          click() {
-            require('electron').shell.openExternal('https://dbgate.org');
-          },
-        },
-        {
-          label: 'DbGate on GitHub',
-          click() {
-            require('electron').shell.openExternal('https://github.com/dbgate/dbgate');
-          },
-        },
-        {
-          label: 'DbGate on docker hub',
-          click() {
-            require('electron').shell.openExternal('https://hub.docker.com/r/dbgate/dbgate');
-          },
-        },
-        {
-          label: 'Report problem or feature request',
-          click() {
-            require('electron').shell.openExternal('https://github.com/dbgate/dbgate/issues/new');
-          },
-        },
-        commandItem('about.show'),
-      ],
-    },
-  ];
+    if (item.command) {
+      return commandItem(item);
+    }
+  });
+
+  template = _cloneDeepWith(template, item => {
+    if (Array.isArray(item) && item.find(x => x.skip)) {
+      return item.filter(x => x && !x.skip);
+    }
+  });
+
+  if (isMac()) {
+    template = [
+      {
+        label: 'DbGate',
+        submenu: [
+          commandItem({ command: 'about.show' }),
+          { role: 'services' },
+          { role: 'hide' },
+          { role: 'hideOthers' },
+          { role: 'unhide' },
+          { role: 'quit' },
+        ],
+      },
+      ...template,
+    ];
+  }
 
   return Menu.buildFromTemplate(template);
 }
@@ -130,29 +125,165 @@ ipcMain.on('update-commands', async (event, arg) => {
     // rebuild menu
     if (menu.label != command.text || menu.accelerator != command.keyText) {
       mainMenu = buildMenu();
-      mainWindow.setMenu(mainMenu);
+
+      Menu.setApplicationMenu(mainMenu);
+      // mainWindow.setMenu(mainMenu);
       return;
     }
 
     menu.enabled = command.enabled;
   }
 });
+ipcMain.on('quit-app', async (event, arg) => {
+  app.quit();
+  if (isMac()) {
+    app.quit();
+  } else {
+    mainWindow.close();
+  }
+});
+ipcMain.on('set-title', async (event, arg) => {
+  mainWindow.setTitle(arg);
+});
+ipcMain.on('open-link', async (event, arg) => {
+  electron.shell.openExternal(arg);
+});
+ipcMain.on('open-dev-tools', () => {
+  mainWindow.webContents.openDevTools();
+});
+ipcMain.on('app-started', async (event, arg) => {
+  if (runCommandOnLoad) {
+    mainWindow.webContents.send('run-command', runCommandOnLoad);
+    runCommandOnLoad = null;
+  }
+});
+ipcMain.on('new-window', async (event, arg) => {
+  createWindow();
+});
+ipcMain.on('window-action', async (event, arg) => {
+  if (!mainWindow) {
+    return;
+  }
+  switch (arg) {
+    case 'minimize':
+      mainWindow.minimize();
+      break;
+    case 'maximize':
+      if (mainWindow.isMaximized()) {
+        mainWindow.unmaximize();
+      } else {
+        mainWindow.maximize();
+      }
+      break;
+    case 'close':
+      mainWindow.close();
+      break;
+    case 'fullscreen-on':
+      mainWindow.setFullScreen(true);
+      break;
+    case 'fullscreen-off':
+      mainWindow.setFullScreen(false);
+      break;
+    case 'devtools':
+      mainWindow.webContents.toggleDevTools();
+      break;
+    case 'reload':
+      mainWindow.webContents.reloadIgnoringCache();
+      break;
+    case 'zoomin':
+      mainWindow.webContents.zoomLevel += 0.5;
+      break;
+    case 'zoomout':
+      mainWindow.webContents.zoomLevel -= 0.5;
+      break;
+    case 'zoomreset':
+      mainWindow.webContents.zoomLevel = 0;
+      break;
+
+    // edit
+    case 'undo':
+      mainWindow.webContents.undo();
+      break;
+    case 'redo':
+      mainWindow.webContents.redo();
+      break;
+    case 'cut':
+      mainWindow.webContents.cut();
+      break;
+    case 'copy':
+      mainWindow.webContents.copy();
+      break;
+    case 'paste':
+      mainWindow.webContents.paste();
+      break;
+  }
+});
+
+ipcMain.handle('showOpenDialog', async (event, { winid, options }) => {
+  const res = electron.dialog.showOpenDialogSync(mainWindow, options);
+  return res;
+});
+ipcMain.handle('showSaveDialog', async (event, { winid, options }) => {
+  const res = electron.dialog.showSaveDialogSync(mainWindow, options);
+  return res;
+});
+ipcMain.handle('showItemInFolder', async (event, { winid, path }) => {
+  electron.shell.showItemInFolder(path);
+});
+ipcMain.handle('openExternal', async (event, { winid, url }) => {
+  electron.shell.openExternal(url);
+});
+
+function fillMissingSettings(value) {
+  const res = {
+    ...value,
+  };
+  if (value['app.useNativeMenu'] !== true && value['app.useNativeMenu'] !== false) {
+    res['app.useNativeMenu'] = false;
+    // res['app.useNativeMenu'] = os.platform() == 'darwin' ? true : false;
+  }
+  return res;
+}
 
 function createWindow() {
-  const bounds = store.get('winBounds');
+  let settingsJson = {};
+  try {
+    const datadir = path.join(os.homedir(), '.dbgate');
+    settingsJson = fillMissingSettings(
+      JSON.parse(fs.readFileSync(path.join(datadir, 'settings.json'), { encoding: 'utf-8' }))
+    );
+  } catch (err) {
+    console.log('Error loading settings.json:', err.message);
+    settingsJson = fillMissingSettings({});
+  }
+
+  const bounds = initialConfig['winBounds'];
+  useNativeMenu = settingsJson['app.useNativeMenu'];
 
   mainWindow = new BrowserWindow({
     width: 1200,
     height: 800,
     title: 'DbGate',
+    frame: useNativeMenu,
+    titleBarStyle: useNativeMenu ? undefined : 'hidden',
     ...bounds,
     icon: os.platform() == 'win32' ? 'icon.ico' : path.resolve(__dirname, '../icon.png'),
+    partition: 'persist:dbgate',
     webPreferences: {
       nodeIntegration: true,
-      enableRemoteModule: true,
+      contextIsolation: false,
+      spellcheck: false,
+      additionalArguments: [`--winid=${++winCounter}`],
     },
   });
 
+  if (initialConfig['winIsMaximized']) {
+    mainWindow.maximize();
+  }
+  if (settingsJson['app.fullscreen']) {
+    mainWindow.setFullScreen(true);
+  }
+
   mainMenu = buildMenu();
   mainWindow.setMenu(mainMenu);
 
@@ -164,55 +295,67 @@ function createWindow() {
         protocol: 'file:',
         slashes: true,
       });
-    mainWindow.webContents.on('did-finish-load', function () {
-      // hideSplash();
-    });
     mainWindow.on('close', () => {
-      store.set('winBounds', mainWindow.getBounds());
+      try {
+        fs.writeFileSync(
+          configRootPath,
+          JSON.stringify({
+            winBounds: mainWindow.getBounds(),
+            winIsMaximized: mainWindow.isMaximized(),
+          }),
+          'utf-8'
+        );
+      } catch (err) {
+        console.log('Error saving config-root:', err.message);
+      }
     });
     mainWindow.loadURL(startUrl);
     if (os.platform() == 'linux') {
       mainWindow.setIcon(path.resolve(__dirname, '../icon.png'));
     }
+    // mainWindow.webContents.toggleDevTools();
   }
 
-  if (process.env.ELECTRON_START_URL) {
-    loadMainWindow();
-  } else {
-    const apiProcess = fork(path.join(__dirname, '../packages/api/dist/bundle.js'), [
-      '--dynport',
-      '--is-electron-bundle',
-      '--native-modules',
-      path.join(__dirname, 'nativeModules'),
-      // '../../../src/nativeModules'
-    ]);
-    apiProcess.on('message', msg => {
-      if (msg.msgtype == 'listening') {
-        const { port, authorization } = msg;
-        global['port'] = port;
-        global['authorization'] = authorization;
-        loadMainWindow();
-      }
-    });
+  if (!apiLoaded) {
+    const apiPackage = path.join(
+      __dirname,
+      process.env.DEVMODE ? '../../packages/api/src/index' : '../packages/api/dist/bundle.js'
+    );
+
+    global.API_PACKAGE = apiPackage;
+    global.NATIVE_MODULES = path.join(__dirname, 'nativeModules');
+
+    // console.log('global.API_PACKAGE', global.API_PACKAGE);
+    const api = require(apiPackage);
+    // console.log(
+    //   'REQUIRED',
+    //   path.resolve(
+    //     path.join(__dirname, process.env.DEVMODE ? '../../packages/api/src/index' : '../packages/api/dist/bundle.js')
+    //   )
+    // );
+    const main = api.getMainModule();
+    main.useAllControllers(null, electron);
+    mainModule = main;
+    apiLoaded = true;
   }
+  mainModule.addElectronSender(mainWindow.webContents);
 
-  // and load the index.html of the app.
-  // mainWindow.loadURL('http://localhost:3000');
-
-  // Open the DevTools.
-  // mainWindow.webContents.openDevTools();
+  loadMainWindow();
 
   // Emitted when the window is closed.
   mainWindow.on('closed', function () {
     // Dereference the window object, usually you would store windows
     // in an array if your app supports multi windows, this is the time
     // when you should delete the corresponding element.
+    mainModule.removeElectronSender(mainWindow.webContents);
     mainWindow = null;
   });
 }
 
 function onAppReady() {
-  autoUpdater.checkForUpdatesAndNotify();
+  if (!process.env.DEVMODE) {
+    autoUpdater.checkForUpdatesAndNotify();
+  }
   createWindow();
 }
 
@@ -225,7 +368,7 @@ app.on('ready', onAppReady);
 app.on('window-all-closed', function () {
   // On OS X it is common for applications and their menu bar
   // to stay active until the user quits explicitly with Cmd + Q
-  if (process.platform !== 'darwin') {
+  if (!isMac()) {
     app.quit();
   }
 });

app/src/mainMenuDefinition.js

@@ -0,0 +1,100 @@
+module.exports = ({ editMenu }) => [
+  {
+    label: 'File',
+    submenu: [
+      { command: 'new.connection', hideDisabled: true },
+      { command: 'new.sqliteDatabase', hideDisabled: true },
+      { command: 'new.window', hideDisabled: true },
+      { divider: true },
+      { command: 'new.query', hideDisabled: true },
+      { command: 'new.freetable', hideDisabled: true },
+      { command: 'new.shell', hideDisabled: true },
+      { command: 'new.jsonl', hideDisabled: true },
+      { divider: true },
+      { command: 'file.open', hideDisabled: true },
+      { command: 'file.openArchive', hideDisabled: true },
+      { divider: true },
+      { command: 'group.save', hideDisabled: true },
+      { command: 'group.saveAs', hideDisabled: true },
+      { divider: true },
+      { command: 'file.exit', hideDisabled: true },
+      { command: 'app.logout', hideDisabled: true, skipInApp: true },
+    ],
+  },
+  {
+    label: 'Window',
+    submenu: [
+      { command: 'tabs.closeTab', hideDisabled: false },
+      { command: 'tabs.closeAll', hideDisabled: false },
+      { command: 'tabs.closeTabsWithCurrentDb', hideDisabled: false },
+      { command: 'tabs.closeTabsButCurrentDb', hideDisabled: false },
+      { divider: true },
+      { command: 'app.zoomIn', hideDisabled: true },
+      { command: 'app.zoomOut', hideDisabled: true },
+      { command: 'app.zoomReset', hideDisabled: true },
+    ],
+  },
+
+  editMenu
+    ? {
+        label: 'Edit',
+        submenu: [
+          { command: 'edit.undo' },
+          { command: 'edit.redo' },
+          { divider: true },
+          { command: 'edit.cut' },
+          { command: 'edit.copy' },
+          { command: 'edit.paste' },
+        ],
+      }
+    : null,
+
+  // {
+  //   label: 'Edit',
+  //   submenu: [
+  //     { role: 'undo' },
+  //     { role: 'redo' },
+  //     { type: 'separator' },
+  //     { role: 'cut' },
+  //     { role: 'copy' },
+  //     { role: 'paste' },
+  //   ],
+  // },
+  {
+    label: 'View',
+    submenu: [
+      { command: 'app.reload', hideDisabled: true },
+      { command: 'app.toggleDevTools', hideDisabled: true },
+      { command: 'app.toggleFullScreen', hideDisabled: true },
+      { command: 'app.minimize', hideDisabled: true },
+      { divider: true },
+      { command: 'theme.changeTheme', hideDisabled: true },
+      { command: 'settings.show' },
+    ],
+  },
+  {
+    label: 'Tools',
+    submenu: [
+      { command: 'database.search', hideDisabled: true },
+      { command: 'commandPalette.show', hideDisabled: true },
+      { command: 'database.switch', hideDisabled: true },
+      { divider: true },
+      { command: 'sql.generator', hideDisabled: true },
+      { command: 'file.import', hideDisabled: true },
+      { command: 'new.modelCompare', hideDisabled: true },
+    ],
+  },
+  {
+    label: 'Help',
+    submenu: [
+      { command: 'app.openDocs', hideDisabled: true },
+      { command: 'app.openWeb', hideDisabled: true },
+      { command: 'app.openIssue', hideDisabled: true },
+      { command: 'app.openSponsoring', hideDisabled: true },
+      { divider: true },
+      { command: 'settings.commands', hideDisabled: true },
+      { command: 'tabs.changelog', hideDisabled: true },
+      { command: 'about.show', hideDisabled: true },
+    ],
+  },
+];

docker-compose.yaml

@@ -0,0 +1,54 @@
+# this compose file is for testing purposes only
+# use it for testing docker containsers built on local machine
+version: "3"
+services:
+  dbgate:
+    build: docker
+    # image: dbgate/dbgate:beta-alpine
+    # image: dbgate/dbgate:alpine
+    # image: dbgate/dbgate:beta
+    restart: always
+    ports:
+      - 3100:3000
+    # volumes:
+    #   - /home/jena/dbgate-data:/root/dbgate-data
+
+    volumes:
+      - dbgate-data:/root/.dbgate
+    
+    # environment:
+    #   WEB_ROOT: /dbgate
+
+      # CONNECTIONS: mssql
+      # LABEL_mssql: MS Sql
+      # SERVER_mssql: mssql
+      # USER_mssql: sa
+      # PORT_mssql: 1433
+      # PASSWORD_mssql: Pwd2020Db
+      # ENGINE_mssql: mssql@dbgate-plugin-mssql
+  # proxy:
+  #   # image: nginx
+  #   build: test/nginx
+  #   ports:
+  #     - 8082:80
+
+    # volumes:
+    #   - /home/jena/test/chinook:/mnt/sqt
+    # environment:
+    #   CONNECTIONS: sqlite
+
+    #   LABEL_sqlite: sqt
+    #   FILE_sqlite: /mnt/sqt/Chinook.db
+    #   ENGINE_sqlite: sqlite@dbgate-plugin-sqlite
+
+  # mssql:
+  #   image: mcr.microsoft.com/mssql/server
+  #   restart: always
+  #   environment:
+  #     - ACCEPT_EULA=Y
+  #     - SA_PASSWORD=Pwd2020Db
+  #     - MSSQL_PID=Express
+
+volumes:
+  dbgate-data:
+    driver: local

docker/.gitignore

@@ -0,0 +1,2 @@
+package.json
+yarn.lock

docker/Dockerfile

@@ -1,9 +1,18 @@
-FROM node:12-alpine
+FROM node:14
+
+RUN apt-get update && apt-get install -y \
+    iputils-ping \
+    iproute2 \
+ && rm -rf /var/lib/apt/lists/*
 
 WORKDIR /home/dbgate-docker
 
 COPY . .
 
+RUN ["chmod", "+x", "/home/dbgate-docker/entrypoint.sh"]
+
 WORKDIR /home/dbgate-docker
 EXPOSE 3000
-CMD node bundle.js
+VOLUME /root/.dbgate
+
+CMD ["/home/dbgate-docker/entrypoint.sh"]

docker/Dockerfile-alpine

@@ -0,0 +1,17 @@
+FROM node:14-alpine
+
+WORKDIR /home/dbgate-docker
+
+RUN apk --no-cache upgrade \
+ && apk --no-cache add  \
+    iputils
+
+COPY . .
+
+RUN ["chmod", "+x", "/home/dbgate-docker/entrypoint.sh"]
+
+WORKDIR /home/dbgate-docker
+EXPOSE 3000
+VOLUME /root/.dbgate
+
+CMD ["/home/dbgate-docker/entrypoint.sh"]

docker/entrypoint.sh

@@ -0,0 +1,11 @@
+#!/bin/sh
+
+HOST_DOMAIN="dockerhost"
+ping -q -c1 $HOST_DOMAIN > /dev/null 2>&1
+if [ $? != 0 ]
+then
+  HOST_IP=$(ip route | awk 'NR==1 {print $3}')
+  echo "$HOST_IP $HOST_DOMAIN" >> /etc/hosts
+fi
+
+node bundle.js

fillNativeModules.js

@@ -5,7 +5,7 @@ let fillContent = '';
 if (process.platform == 'win32') {
   fillContent += `content.msnodesqlv8 = () => require('msnodesqlv8');`;
 }
-fillContent += `content['better-sqlite3-with-prebuilds'] = () => require('better-sqlite3-with-prebuilds');`;
+fillContent += `content['better-sqlite3'] = () => require('better-sqlite3');`;
 
 const getContent = (empty) => `
 // this file is generated automatically by script fillNativeModules.js, do not edit it manually

integration-tests/__tests__/alter-database.spec.js

@@ -0,0 +1,68 @@
+const stableStringify = require('json-stable-stringify');
+const _ = require('lodash');
+const fp = require('lodash/fp');
+const uuidv1 = require('uuid/v1');
+const { testWrapper } = require('../tools');
+const engines = require('../engines');
+const { getAlterDatabaseScript, extendDatabaseInfo, generateDbPairingId } = require('dbgate-tools');
+
+function flatSource() {
+  return _.flatten(
+    engines.map(engine => (engine.objects || []).map(object => [engine.label, object.type, object, engine]))
+  );
+}
+
+async function testDatabaseDiff(conn, driver, mangle, createObject = null) {
+  await driver.query(conn, `create table t1 (id int not null primary key)`);
+
+  await driver.query(
+    conn,
+    `create table t2 (
+    id int not null primary key, 
+    t1_id int null references t1(id)
+  )`
+  );
+
+  if (createObject) await driver.query(conn, createObject);
+
+  const structure1 = generateDbPairingId(extendDatabaseInfo(await driver.analyseFull(conn)));
+  let structure2 = _.cloneDeep(structure1);
+  mangle(structure2);
+  structure2 = extendDatabaseInfo(structure2);
+
+  const { sql } = getAlterDatabaseScript(structure1, structure2, {}, structure1, structure2, driver);
+  console.log('RUNNING ALTER SQL', driver.engine, ':', sql);
+
+  await driver.script(conn, sql);
+
+  const structure2Real = extendDatabaseInfo(await driver.analyseFull(conn));
+
+  expect(structure2Real.tables.length).toEqual(structure2.tables.length);
+  return structure2Real;
+}
+
+describe('Alter database', () => {
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Drop referenced table - %s',
+    testWrapper(async (conn, driver, engine) => {
+      await testDatabaseDiff(conn, driver, db => {
+        _.remove(db.tables, x => x.pureName == 't1');
+      });
+    })
+  );
+
+  test.each(flatSource())(
+    'Drop object - %s - %s',
+    testWrapper(async (conn, driver, type, object, engine) => {
+      const db = await testDatabaseDiff(
+        conn,
+        driver,
+        db => {
+          _.remove(db[type], x => x.pureName == 'obj1');
+        },
+        object.create1
+      );
+      expect(db[type].length).toEqual(0);
+    })
+  );
+});

integration-tests/__tests__/alter-table.spec.js

@@ -46,7 +46,7 @@ async function testTableDiff(conn, driver, mangle) {
   mangle(tget(structure2));
   structure2 = extendDatabaseInfo(structure2);
 
-  const sql = getAlterTableScript(tget(structure1), tget(structure2), {}, structure2, driver);
+  const { sql } = getAlterTableScript(tget(structure1), tget(structure2), {}, structure1, structure2, driver);
   console.log('RUNNING ALTER SQL', driver.engine, ':', sql);
 
   await driver.script(conn, sql);
@@ -62,12 +62,13 @@ const TESTED_COLUMNS = ['col_pk', 'col_std', 'col_def', 'col_fk', 'col_ref', 'co
 // const TESTED_COLUMNS = ['col_idx'];
 // const TESTED_COLUMNS = ['col_def'];
 // const TESTED_COLUMNS = ['col_std'];
+// const TESTED_COLUMNS = ['col_ref'];
 
 function engines_columns_source() {
   return _.flatten(engines.map(engine => TESTED_COLUMNS.map(column => [engine.label, column, engine])));
 }
 
-describe('Alter processor', () => {
+describe('Alter table', () => {
   test.each(engines.map(engine => [engine.label, engine]))(
     'Add column - %s',
     testWrapper(async (conn, driver, engine) => {

integration-tests/__tests__/db-import.spec.js

@@ -0,0 +1,75 @@
+const engines = require('../engines');
+const stream = require('stream');
+const { testWrapper } = require('../tools');
+const tableWriter = require('dbgate-api/src/shell/tableWriter');
+const copyStream = require('dbgate-api/src/shell/copyStream');
+const fakeObjectReader = require('dbgate-api/src/shell/fakeObjectReader');
+
+function createImportStream() {
+  const pass = new stream.PassThrough({
+    objectMode: true,
+  });
+  pass.write({ columns: [{ columnName: 'id' }, { columnName: 'country' }], __isStreamHeader: true });
+  pass.write({ id: 1, country: 'Czechia' });
+  pass.write({ id: 2, country: 'Austria' });
+  pass.write({ country: 'Germany', id: 3 });
+  pass.write({ country: 'Romania', id: 4 });
+  pass.write({ country: 'Great Britain', id: 5 });
+  pass.write({ country: 'Bosna, Hecegovina', id: 6 });
+  pass.end();
+
+  return pass;
+}
+
+describe('DB Import', () => {
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Import one table - %s',
+    testWrapper(async (conn, driver, engine) => {
+      // const reader = await fakeObjectReader({ delay: 10 });
+      // const reader = await fakeObjectReader();
+      const reader = createImportStream();
+      const writer = await tableWriter({
+        systemConnection: conn,
+        driver,
+        pureName: 't1',
+        createIfNotExists: true,
+      });
+      await copyStream(reader, writer);
+
+      const res = await driver.query(conn, `select count(*) as cnt from t1`);
+      expect(res.rows[0].cnt.toString()).toEqual('6');
+    })
+  );
+
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Import two tables - %s',
+    testWrapper(async (conn, driver, engine) => {
+      // const reader = await fakeObjectReader({ delay: 10 });
+      // const reader = await fakeObjectReader();
+      const reader1 = createImportStream();
+      const writer1 = await tableWriter({
+        systemConnection: conn,
+        driver,
+        pureName: 't1',
+        createIfNotExists: true,
+      });
+      await copyStream(reader1, writer1);
+
+      const reader2 = createImportStream();
+      const writer2 = await tableWriter({
+        systemConnection: conn,
+        driver,
+        pureName: 't2',
+        createIfNotExists: true,
+      });
+      await copyStream(reader2, writer2);
+
+      const res1 = await driver.query(conn, `select count(*) as cnt from t1`);
+      expect(res1.rows[0].cnt.toString()).toEqual('6');
+
+      const res2 = await driver.query(conn, `select count(*) as cnt from t2`);
+      expect(res2.rows[0].cnt.toString()).toEqual('6');
+    })
+  );
+
+});

integration-tests/__tests__/deploy-database.spec.js

@@ -0,0 +1,300 @@
+/// TODO
+
+const { testWrapper } = require('../tools');
+const _ = require('lodash');
+const engines = require('../engines');
+const deployDb = require('dbgate-api/src/shell/deployDb');
+const { databaseInfoFromYamlModel } = require('dbgate-tools');
+const generateDeploySql = require('dbgate-api/src/shell/generateDeploySql');
+
+function checkStructure(structure, model) {
+  const expected = databaseInfoFromYamlModel(model);
+  expect(structure.tables.length).toEqual(expected.tables.length);
+
+  for (const [realTable, expectedTable] of _.zip(
+    _.sortBy(structure.tables, 'pureName'),
+    _.sortBy(expected.tables, 'pureName')
+  )) {
+    expect(realTable.columns.length).toBeGreaterThanOrEqual(expectedTable.columns.length);
+  }
+}
+
+async function testDatabaseDeploy(conn, driver, dbModelsYaml, testEmptyLastScript) {
+  let index = 0;
+  for (const loadedDbModel of dbModelsYaml) {
+    const { sql, isEmpty } = await generateDeploySql({
+      systemConnection: conn,
+      driver,
+      loadedDbModel,
+    });
+    console.debug('Generated deploy script:', sql);
+    expect(sql.toUpperCase().includes('DROP ')).toBeFalsy();
+
+    console.log('dbModelsYaml.length', dbModelsYaml.length, index);
+    if (testEmptyLastScript && index == dbModelsYaml.length - 1) {
+      expect(isEmpty).toBeTruthy();
+    }
+
+    await deployDb({
+      systemConnection: conn,
+      driver,
+      loadedDbModel,
+    });
+
+    index++;
+  }
+
+  const structure = await driver.analyseFull(conn);
+  checkStructure(structure, dbModelsYaml[dbModelsYaml.length - 1]);
+}
+
+describe('Deploy database', () => {
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Deploy database simple - %s',
+    testWrapper(async (conn, driver, engine) => {
+      await testDatabaseDeploy(conn, driver, [
+        [
+          {
+            name: 't1.table.yaml',
+            json: {
+              name: 't1',
+              columns: [{ name: 'id', type: 'int' }],
+              primaryKey: ['id'],
+            },
+          },
+        ],
+      ]);
+    })
+  );
+
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Deploy database simple twice - %s',
+    testWrapper(async (conn, driver, engine) => {
+      await testDatabaseDeploy(
+        conn,
+        driver,
+        [
+          [
+            {
+              name: 't1.table.yaml',
+              json: {
+                name: 't1',
+                columns: [{ name: 'id', type: 'int' }],
+                primaryKey: ['id'],
+              },
+            },
+          ],
+          [
+            {
+              name: 't1.table.yaml',
+              json: {
+                name: 't1',
+                columns: [{ name: 'id', type: 'int' }],
+                primaryKey: ['id'],
+              },
+            },
+          ],
+        ],
+        true
+      );
+    })
+  );
+
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Add column - %s',
+    testWrapper(async (conn, driver, engine) => {
+      await testDatabaseDeploy(conn, driver, [
+        [
+          {
+            name: 't1.table.yaml',
+            json: {
+              name: 't1',
+              columns: [{ name: 'id', type: 'int' }],
+              primaryKey: ['id'],
+            },
+          },
+        ],
+        [
+          {
+            name: 't1.table.yaml',
+            json: {
+              name: 't1',
+              columns: [
+                { name: 'id', type: 'int' },
+                { name: 'val', type: 'int' },
+              ],
+              primaryKey: ['id'],
+            },
+          },
+        ],
+      ]);
+    })
+  );
+
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Dont drop column - %s',
+    testWrapper(async (conn, driver, engine) => {
+      await testDatabaseDeploy(
+        conn,
+        driver,
+        [
+          [
+            {
+              name: 't1.table.yaml',
+              json: {
+                name: 't1',
+                columns: [
+                  { name: 'id', type: 'int' },
+                  { name: 'val', type: 'int' },
+                ],
+                primaryKey: ['id'],
+              },
+            },
+          ],
+          [
+            {
+              name: 't1.table.yaml',
+              json: {
+                name: 't1',
+                columns: [{ name: 'id', type: 'int' }],
+                primaryKey: ['id'],
+              },
+            },
+          ],
+        ],
+        true
+      );
+    })
+  );
+
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Foreign keys - %s',
+    testWrapper(async (conn, driver, engine) => {
+      await testDatabaseDeploy(
+        conn,
+        driver,
+        [
+          [
+            {
+              name: 't2.table.yaml',
+              json: {
+                name: 't2',
+                columns: [
+                  { name: 't2id', type: 'int' },
+                  { name: 't1id', type: 'int', references: 't1' },
+                ],
+                primaryKey: ['t2id'],
+              },
+            },
+            {
+              name: 't1.table.yaml',
+              json: {
+                name: 't1',
+                columns: [{ name: 't1id', type: 'int' }],
+                primaryKey: ['t1id'],
+              },
+            },
+          ],
+          [
+            {
+              name: 't2.table.yaml',
+              json: {
+                name: 't2',
+                columns: [
+                  { name: 't2id', type: 'int' },
+                  { name: 't1id', type: 'int', references: 't1' },
+                ],
+                primaryKey: ['t2id'],
+              },
+            },
+            {
+              name: 't1.table.yaml',
+              json: {
+                name: 't1',
+                columns: [{ name: 't1id', type: 'int' }],
+                primaryKey: ['t1id'],
+              },
+            },
+          ],
+        ],
+        true
+      );
+    })
+  );
+
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Deploy preloaded data - %s',
+    testWrapper(async (conn, driver, engine) => {
+      await testDatabaseDeploy(conn, driver, [
+        [
+          {
+            name: 't1.table.yaml',
+            json: {
+              name: 't1',
+              columns: [
+                { name: 'id', type: 'int' },
+                { name: 'value', type: 'int' },
+              ],
+              primaryKey: ['id'],
+              data: [
+                { id: 1, value: 1 },
+                { id: 2, value: 2 },
+                { id: 3, value: 3 },
+              ],
+            },
+          },
+        ],
+      ]);
+
+      const res = await driver.query(conn, `select count(*) as cnt from t1`);
+      expect(res.rows[0].cnt.toString()).toEqual('3');
+    })
+  );
+
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Deploy preloaded data - update - %s',
+    testWrapper(async (conn, driver, engine) => {
+      await testDatabaseDeploy(conn, driver, [
+        [
+          {
+            name: 't1.table.yaml',
+            json: {
+              name: 't1',
+              columns: [
+                { name: 'id', type: 'int' },
+                { name: 'val', type: 'int' },
+              ],
+              primaryKey: ['id'],
+              data: [
+                { id: 1, val: 1 },
+                { id: 2, val: 2 },
+                { id: 3, val: 3 },
+              ],
+            },
+          },
+        ],
+        [
+          {
+            name: 't1.table.yaml',
+            json: {
+              name: 't1',
+              columns: [
+                { name: 'id', type: 'int' },
+                { name: 'val', type: 'int' },
+              ],
+              primaryKey: ['id'],
+              data: [
+                { id: 1, val: 1 },
+                { id: 2, val: 5 },
+                { id: 3, val: 3 },
+              ],
+            },
+          },
+        ],
+      ]);
+
+      const res = await driver.query(conn, `select val from t1 where id = 2`);
+      expect(res.rows[0].val.toString()).toEqual('5');
+    })
+  );
+});

integration-tests/__tests__/table-analyse.spec.js

@@ -14,7 +14,7 @@ const txMatch = (tname, vcolname, nextcol) =>
       expect.objectContaining({
         columnName: 'id',
         notNull: true,
-        dataType: expect.stringContaining('int'),
+        dataType: expect.stringMatching(/int/i),
       }),
       expect.objectContaining({
         columnName: vcolname,

integration-tests/docker-compose.yaml

@@ -8,30 +8,39 @@ services:
     ports: 
       - 15000:5432
 
-  mysql:
-    image: mysql:8.0.18
+  mariadb:
+    image: mariadb
     command: --default-authentication-plugin=mysql_native_password
     restart: always
     ports: 
-      - 15001:3306
+      - 15004:3306
     environment:
       - MYSQL_ROOT_PASSWORD=Pwd2020Db
 
-  mssql:
-    image: mcr.microsoft.com/mssql/server
-    restart: always
-    ports: 
-      - 15002:1433
-    environment:
-      - ACCEPT_EULA=Y
-      - SA_PASSWORD=Pwd2020Db
-      - MSSQL_PID=Express
+  # mysql:
+  #   image: mysql:8.0.18
+  #   command: --default-authentication-plugin=mysql_native_password
+  #   restart: always
+  #   ports: 
+  #     - 15001:3306
+  #   environment:
+  #     - MYSQL_ROOT_PASSWORD=Pwd2020Db
+
+  # mssql:
+  #   image: mcr.microsoft.com/mssql/server
+  #   restart: always
+  #   ports: 
+  #     - 15002:1433
+  #   environment:
+  #     - ACCEPT_EULA=Y
+  #     - SA_PASSWORD=Pwd2020Db
+  #     - MSSQL_PID=Express
       
-  cockroachdb:
-    image: cockroachdb/cockroach
-    ports:
-      - 15003:26257
-    command: start-single-node --insecure
+  # cockroachdb:
+  #   image: cockroachdb/cockroach
+  #   ports:
+  #     - 15003:26257
+  #   command: start-single-node --insecure
 
 #   mongodb:
 #     image: mongo:4.0.12

integration-tests/engines.js

@@ -31,6 +31,23 @@ const engines = [
     objects: [views],
     dbSnapshotBySeconds: true,
   },
+  {
+    label: 'MariaDB',
+    connection: {
+      engine: 'mariadb@dbgate-plugin-mysql',
+      password: 'Pwd2020Db',
+      user: 'root',
+      server: 'mysql',
+      port: 3306,
+    },
+    local: {
+      server: 'localhost',
+      port: 15004,
+    },
+    skipOnCI: true,
+    objects: [views],
+    dbSnapshotBySeconds: true,
+  },
   {
     label: 'PostgreSQL',
     connection: {
@@ -116,11 +133,12 @@ const engines = [
 
 const filterLocal = [
   // filter local testing
-  'MySQL',
+  '-MySQL',
+  '-MariaDB',
   'PostgreSQL',
-  'SQL Server',
-  'SQLite',
-  'CockroachDB',
+  '-SQL Server',
+  '-SQLite',
+  '-CockroachDB',
 ];
 
 module.exports = process.env.CITEST

integration-tests/package.json

@@ -1,6 +1,6 @@
 {
   "name": "dbgate-integration-tests",
-  "version": "4.1.1",
+  "version": "5.0.0-alpha.1",
   "homepage": "https://dbgate.org/",
   "repository": {
     "type": "git",

misc/convert-icons.sh

@@ -0,0 +1,106 @@
+# magick icon.svg -resize 1024x1024 -transparent white -background transparent app/icon1024.png
+
+# magick app/icon1024.png -resize 512x512 app/icon512.png
+# magick app/icon1024.png -resize 256x256 app/icon.png
+# magick app/icon1024.png -resize 32x32 app/icon32.png
+# magick icon.svg -define icon:auto-resize="256,128,96,64,48,32,16" -transparent white -background transparent app/icon.ico
+
+# # magick icon.svg -resize 512x512 -transparent white -background transparent app/icon512.png
+# # magick icon.svg -resize 256x256 -transparent white -background transparent app/icon.png
+# # magick icon.svg -resize 32x32 -transparent white -background transparent app/icon32.png
+# # magick icon.svg -define icon:auto-resize="256,128,96,64,48,32,16" -transparent white -background transparent app/icon.ico
+
+# magick app/icon1024.png -resize 512x512 app/icons/512x512.png
+# magick app/icon1024.png -resize 256x256 app/icons/256x256.png
+# magick app/icon1024.png -resize 128x128 app/icons/128x128.png
+# magick app/icon1024.png -resize 64x64 app/icons/64x64.png
+# magick app/icon1024.png -resize 48x48 app/icons/48x48.png
+# magick app/icon1024.png -resize 32x32 app/icons/32x32.png
+# magick app/icon1024.png -resize 16x16 app/icons/16x16.png
+
+# # magick icon.svg -resize 16x16 -transparent white -background transparent app/icons/16x16.png
+# # magick icon.svg -resize 32x32 -transparent white -background transparent app/icons/32x32.png
+# # magick icon.svg -resize 48x48 -transparent white -background transparent app/icons/48x48.png
+# # magick icon.svg -resize 64x64 -transparent white -background transparent app/icons/64x64.png
+# # magick icon.svg -resize 128x128 -transparent white -background transparent app/icons/128x128.png
+# # magick icon.svg -resize 256x256 -transparent white -background transparent app/icons/256x256.png
+# # magick icon.svg -resize 512x512 -transparent white -background transparent app/icons/512x512.png
+
+# magick icon.svg -resize 1024x1024 icon.png
+# magick icon.svg -resize 1024x1024 -transparent white -background transparent icon.png
+
+STROKE_WIDTH=30
+LEFT=150
+RIGHT=850
+
+
+
+magick  \
+    \( \
+        -size 1000x1000 -define gradient:direction=east 'gradient:#0050b3-#1890ff' \
+        \( +clone -fill Black -colorize 100 \
+        -fill White -stroke White -draw "arc $LEFT,750 $RIGHT,950 0,360" -draw "rectangle $LEFT,150 $RIGHT,850" \
+        \) \
+        -alpha off \
+        -compose CopyOpacity -composite \
+    \) \
+    \( \
+        -size 1000x1000 -define gradient:direction=east 'gradient:#096dd9-#40a9ff' \
+        \( +clone -fill Black -colorize 100 \
+        -fill White -draw "arc $LEFT,50 $RIGHT,250 0,360" \
+        \) \
+        -alpha off \
+        -compose CopyOpacity -composite \
+    \) \
+    -compose Over -composite \
+    -strokewidth $STROKE_WIDTH -stroke '#0050b3' -fill transparent \
+    -draw "arc $LEFT,225 $RIGHT,425  0,180" \
+    -draw "arc $LEFT,400 $RIGHT,600  0,180" \
+    -draw "arc $LEFT,575 $RIGHT,775  0,180" \
+    -draw "arc $LEFT,750 $RIGHT,950  0,180" \
+    -draw "arc $LEFT,50 $RIGHT,250  0,360" \
+    -draw "line $LEFT,150 $LEFT,850" \
+    -draw "line $RIGHT,150 $RIGHT,850" \
+    -fill '#fafafa' -stroke '#8c8c8c' -strokewidth 3 \
+    -pointsize 800 -font './Mcbungus-Regular.ttf' \
+    -gravity center \
+    -draw 'text 0,100 "G"' \
+    icon.png
+
+
+# magick \
+#   \( \
+#     -size 300x300 gradient:red-blue \
+#     \( +clone -fill Black -colorize 100 \
+#       -fill White -draw "polygon 50,50 250,50 200,200" \
+#     \) \
+#     -alpha off \
+#     -compose CopyOpacity -composite \
+#   \) \
+#   \( \
+#     -size 300x300 'gradient:#f80-#08f' \
+#     \( +clone -fill Black -colorize 100 \
+#       -fill White -draw "polygon 50,150 250,150 200,300" \
+#     \) \
+#     -alpha off \
+#     -compose CopyOpacity -composite \
+#   \) \
+#   -compose Over -composite \
+#   icon.png    
+
+magick icon.png -resize 512x512! ../app/icon512.png
+magick icon.png -resize 256x256! ../app/icon.png
+magick icon.png -resize 32x32! ../app/icon32.png
+magick icon.png -define icon:auto-resize="256,128,96,64,48,32,16" ../app/icon.ico
+
+magick icon.png -resize 512x512! ../app/icons/512x512.png
+magick icon.png -resize 256x256! ../app/icons/256x256.png
+magick icon.png -resize 128x128! ../app/icons/128x128.png
+magick icon.png -resize 64x64! ../app/icons/64x64.png
+magick icon.png -resize 48x48! ../app/icons/48x48.png
+magick icon.png -resize 32x32! ../app/icons/32x32.png
+magick icon.png -resize 16x16! ../app/icons/16x16.png
+
+magick icon.png -resize 192x192! ../packages/web/public/logo192.png
+magick icon.png -resize 512x512! ../packages/web/public/logo512.png
+magick icon.png -define icon:auto-resize="256,128,96,64,48,32,16" ../packages/web/public/favicon.ico

package.json

@@ -1,6 +1,6 @@
 {
   "private": true,
-  "version": "4.2.8-beta.3",
+  "version": "5.0.2",
   "name": "dbgate-all",
   "workspaces": [
     "packages/*",
@@ -9,22 +9,20 @@
   ],
   "scripts": {
     "start:api": "yarn workspace dbgate-api start",
+    "start:app": "cd app && yarn start",
     "start:api:portal": "yarn workspace dbgate-api start:portal",
-    "start:api:covid": "yarn workspace dbgate-api start:covid",
+    "start:api:singledb": "yarn workspace dbgate-api start:singledb",
     "start:web": "yarn workspace dbgate-web dev",
     "start:sqltree": "yarn workspace dbgate-sqltree start",
     "start:tools": "yarn workspace dbgate-tools start",
     "start:datalib": "yarn workspace dbgate-datalib start",
     "start:filterparser": "yarn workspace dbgate-filterparser start",
-    "start:querysplitter": "yarn workspace dbgate-query-splitter start",
     "build:sqltree": "yarn workspace dbgate-sqltree build",
     "build:datalib": "yarn workspace dbgate-datalib build",
     "build:filterparser": "yarn workspace dbgate-filterparser build",
-    "build:querysplitter": "yarn workspace dbgate-query-splitter build",
     "build:tools": "yarn workspace dbgate-tools build",
-    "build:lib": "yarn build:querysplitter && yarn build:tools && yarn build:sqltree && yarn build:filterparser && yarn build:datalib",
+    "build:lib": "yarn build:sqltree && yarn build:tools && yarn build:filterparser && yarn build:datalib",
     "build:app": "yarn plugins:copydist && cd app && yarn install && yarn build",
-    "build:app:mac": "yarn plugins:copydist && cd app && yarn install && yarn build:mac",
     "build:api": "yarn workspace dbgate-api build",
     "build:web:docker": "yarn workspace dbgate-web build",
     "build:plugins:frontend": "workspaces-run --only=\"dbgate-plugin-*\" -- yarn build:frontend",
@@ -40,18 +38,19 @@
     "resetPackagedPlugins": "node resetPackagedPlugins",
     "prettier": "prettier --write packages/api/src && prettier --write packages/datalib/src && prettier --write packages/filterparser/src && prettier --write packages/sqltree/src && prettier --write packages/tools/src && prettier --write packages/types && prettier --write packages/web/src && prettier --write app/src",
     "copy:docker:build": "copyfiles packages/api/dist/* docker -f && copyfiles packages/web/public/* docker -u 2 && copyfiles \"packages/web/public/**/*\" docker -u 2 && copyfiles \"plugins/dist/**/*\" docker/plugins -u 2",
-    "prepare:docker": "yarn plugins:copydist && yarn build:web:docker && yarn build:api && yarn copy:docker:build",
+    "install:sqlite:docker": "cd docker && yarn init --yes && yarn add better-sqlite3 && cd ..",
+    "prepare:docker": "yarn plugins:copydist && yarn build:web:docker && yarn build:api && yarn copy:docker:build && yarn install:sqlite:docker",
     "start": "concurrently --kill-others-on-fail \"yarn start:api\" \"yarn start:web\"",
-    "lib": "concurrently --kill-others-on-fail \"yarn start:sqltree\" \"yarn start:filterparser\" \"yarn start:datalib\" \"yarn start:tools\" \"yarn start:querysplitter\" \"yarn build:plugins:frontend:watch\"",
+    "lib": "concurrently --kill-others-on-fail \"yarn start:sqltree\" \"yarn start:filterparser\" \"yarn start:datalib\" \"yarn start:tools\" \"yarn build:plugins:frontend:watch\"",
     "ts:api": "yarn workspace dbgate-api ts",
     "ts:web": "yarn workspace dbgate-web ts",
     "ts": "yarn ts:api && yarn ts:web",
-    "postinstall": "yarn resetPackagedPlugins && yarn build:lib && patch-package && yarn fillNativeModules && yarn build:plugins:frontend"
+    "postinstall": "yarn resetPackagedPlugins && yarn build:lib && patch-package && yarn fillNativeModules && yarn build:plugins:frontend",
+    "dbgate-serve": "node packages/dbgate/bin/dbgate-serve.js"
   },
   "dependencies": {
     "concurrently": "^5.1.0",
-    "patch-package": "^6.2.1",
-    "socket.io": "^2.3.0"
+    "patch-package": "^6.2.1"
   },
   "devDependencies": {
     "copyfiles": "^2.2.0",

packages/api/.env

@@ -1 +1,15 @@
 DEVMODE=1
+# PERMISSIONS=~widgets/app,~widgets/plugins
+# DISABLE_SHELL=1
+# HIDE_APP_EDITOR=1
+
+
+# DEVWEB=1
+# LOGINS=admin,test
+
+# LOGIN_PASSWORD_admin=admin
+# LOGIN_PERMISSIONS_admin=*
+
+# LOGIN_PASSWORD_test=test
+# LOGIN_PERMISSIONS_test=~*, widgets/database
+# WORKSPACE_DIR=/home/jena/dbgate-data-2

packages/api/.env-portal

@@ -1,17 +0,0 @@
-DEVMODE=1
-
-CONNECTIONS=mysql,postgres
-
-LABEL_mysql=MySql localhost
-SERVER_mysql=localhost
-USER_mysql=root
-PASSWORD_mysql=test
-PORT_mysql=3307
-ENGINE_mysql=mysql@dbgate-plugin-mysql
-
-LABEL_postgres=Postgres localhost
-SERVER_postgres=localhost
-USER_postgres=postgres
-PASSWORD_postgres=test
-PORT_postgres=5433
-ENGINE_postgres=postgres@dbgate-plugin-postgres

packages/api/env/portal/.env

@@ -0,0 +1,51 @@
+DEVMODE=1
+
+CONNECTIONS=mysql,postgres,mongo,mongo2,mysqlssh,sqlite,relational
+
+LABEL_mysql=MySql localhost
+SERVER_mysql=localhost
+USER_mysql=root
+PASSWORD_mysql=test
+PORT_mysql=3307
+ENGINE_mysql=mysql@dbgate-plugin-mysql
+
+LABEL_postgres=Postgres localhost
+SERVER_postgres=localhost
+USER_postgres=postgres
+PASSWORD_postgres=test
+PORT_postgres=5433
+ENGINE_postgres=postgres@dbgate-plugin-postgres
+
+LABEL_mongo=Mongo URL
+URL_mongo=mongodb://localhost:27017
+ENGINE_mongo=mongo@dbgate-plugin-mongo
+
+LABEL_mongo2=Mongo Server
+SERVER_mongo2=localhost
+ENGINE_mongo2=mongo@dbgate-plugin-mongo
+
+LABEL_mysqlssh=MySql SSH
+SERVER_mysqlssh=localhost
+USER_mysqlssh=root
+PASSWORD_mysqlssh=xxx
+PORT_mysqlssh=3316
+ENGINE_mysqlssh=mysql@dbgate-plugin-mysql
+USE_SSH_mysqlssh=1
+SSH_HOST_mysqlssh=demo.dbgate.org
+SSH_PORT_mysqlssh=22
+SSH_MODE_mysqlssh=userPassword
+SSH_LOGIN_mysqlssh=root
+SSH_PASSWORD_mysqlssh=xxx
+
+LABEL_sqlite=sqlite
+FILE_sqlite=/home/jena/.dbgate/files/sqlite/feeds.sqlite
+ENGINE_sqlite=sqlite@dbgate-plugin-sqlite
+
+LABEL_relational=Relational dataset repo
+SERVER_relational=relational.fit.cvut.cz
+USER_relational=guest
+PASSWORD_relational=relational
+ENGINE_relational=mariadb@dbgate-plugin-mysql
+READONLY_relational=1
+
+# docker run -p 3000:3000 -e CONNECTIONS=mongo -e URL_mongo=mongodb://localhost:27017 -e ENGINE_mongo=mongo@dbgate-plugin-mongo -e LABEL_mongo=mongo dbgate/dbgate:beta

packages/api/env/singledb/.env

@@ -8,6 +8,8 @@ USER_mysql=root
 PASSWORD_mysql=test
 PORT_mysql=3307
 ENGINE_mysql=mysql@dbgate-plugin-mysql
+DBCONFIG_mysql=[{"name":"Chinook","connectionColor":"cyan"}]
+
 
 SINGLE_CONNECTION=mysql
 SINGLE_DATABASE=Chinook

packages/api/package.json

@@ -1,7 +1,7 @@
 {
   "name": "dbgate-api",
   "main": "src/index.js",
-  "version": "4.1.1",
+  "version": "5.0.0-alpha.1",
   "homepage": "https://dbgate.org/",
   "repository": {
     "type": "git",
@@ -19,15 +19,17 @@
   "dependencies": {
     "async-lock": "^1.2.4",
     "axios": "^0.21.1",
-    "better-sqlite3-with-prebuilds": "^7.1.8",
     "body-parser": "^1.19.0",
     "bufferutil": "^4.0.1",
     "byline": "^5.0.0",
     "compare-versions": "^3.6.0",
     "cors": "^2.8.5",
     "cross-env": "^6.0.3",
-    "dbgate-sqltree": "^4.1.1",
-    "dbgate-tools": "^4.1.1",
+    "dbgate-query-splitter": "^4.9.0",
+    "dbgate-sqltree": "^5.0.0-alpha.1",
+    "dbgate-tools": "^5.0.0-alpha.1",
+    "diff": "^5.0.0",
+    "diff2html": "^3.4.13",
     "eslint": "^6.8.0",
     "express": "^4.17.1",
     "express-basic-auth": "^1.2.0",
@@ -36,23 +38,23 @@
     "fs-reverse": "^0.0.3",
     "get-port": "^5.1.1",
     "http": "^0.0.0",
+    "is-electron": "^2.2.1",
+    "js-yaml": "^4.1.0",
     "json-stable-stringify": "^1.0.1",
     "line-reader": "^0.4.0",
     "lodash": "^4.17.21",
     "ncp": "^2.0.0",
-    "nedb-promises": "^4.0.1",
     "node-cron": "^2.0.3",
     "node-ssh-forward": "^0.7.2",
     "portfinder": "^1.0.28",
     "simple-encryptor": "^4.0.0",
-    "socket.io": "^2.3.0",
     "tar": "^6.0.5",
     "uuid": "^3.4.0"
   },
   "scripts": {
     "start": "env-cmd node src/index.js",
-    "start:portal": "env-cmd -f .env-portal node src/index.js",
-    "start:singledb": "env-cmd -f .env-singledb node src/index.js",
+    "start:portal": "env-cmd -f env/portal/.env node src/index.js",
+    "start:singledb": "env-cmd -f env/singledb/.env node src/index.js",
     "start:filedb": "env-cmd node src/index.js /home/jena/test/chinook/Chinook.db",
     "start:singleconn": "env-cmd node src/index.js --server localhost --user root --port 3307 --engine mysql@dbgate-plugin-mysql --password test",
     "ts": "tsc",
@@ -61,15 +63,16 @@
   "devDependencies": {
     "@types/fs-extra": "^9.0.11",
     "@types/lodash": "^4.14.149",
-    "dbgate-types": "^4.1.1",
+    "dbgate-types": "^5.0.0-alpha.1",
     "env-cmd": "^10.1.0",
     "node-loader": "^1.0.2",
     "nodemon": "^2.0.2",
-    "typescript": "^3.7.4",
+    "typescript": "^4.4.3",
     "webpack": "^4.42.0",
     "webpack-cli": "^3.3.11"
   },
   "optionalDependencies": {
-    "msnodesqlv8": "^2.0.10"
+    "better-sqlite3": "7.5.0",
+    "msnodesqlv8": "^2.4.4"
   }
 }

packages/api/src/controllers/apps.js

@@ -0,0 +1,280 @@
+const fs = require('fs-extra');
+const _ = require('lodash');
+const path = require('path');
+const { appdir } = require('../utility/directories');
+const socket = require('../utility/socket');
+const connections = require('./connections');
+
+module.exports = {
+  folders_meta: true,
+  async folders() {
+    const folders = await fs.readdir(appdir());
+    return [
+      ...folders.map(name => ({
+        name,
+      })),
+    ];
+  },
+
+  createFolder_meta: true,
+  async createFolder({ folder }) {
+    const name = await this.getNewAppFolder({ name: folder });
+    await fs.mkdir(path.join(appdir(), name));
+    socket.emitChanged('app-folders-changed');
+    this.emitChangedDbApp(folder);
+    return name;
+  },
+
+  files_meta: true,
+  async files({ folder }) {
+    if (!folder) return [];
+    const dir = path.join(appdir(), folder);
+    if (!(await fs.exists(dir))) return [];
+    const files = await fs.readdir(dir);
+
+    function fileType(ext, type) {
+      return files
+        .filter(name => name.endsWith(ext))
+        .map(name => ({
+          name: name.slice(0, -ext.length),
+          label: path.parse(name.slice(0, -ext.length)).base,
+          type,
+        }));
+    }
+
+    return [
+      ...fileType('.command.sql', 'command.sql'),
+      ...fileType('.query.sql', 'query.sql'),
+      ...fileType('.config.json', 'config.json'),
+    ];
+  },
+
+  async emitChangedDbApp(folder) {
+    const used = await this.getUsedAppFolders();
+    if (used.includes(folder)) {
+      socket.emitChanged('used-apps-changed');
+    }
+  },
+
+  refreshFiles_meta: true,
+  async refreshFiles({ folder }) {
+    socket.emitChanged(`app-files-changed-${folder}`);
+  },
+
+  refreshFolders_meta: true,
+  async refreshFolders() {
+    socket.emitChanged(`app-folders-changed`);
+  },
+
+  deleteFile_meta: true,
+  async deleteFile({ folder, file, fileType }) {
+    await fs.unlink(path.join(appdir(), folder, `${file}.${fileType}`));
+    socket.emitChanged(`app-files-changed-${folder}`);
+    this.emitChangedDbApp(folder);
+  },
+
+  renameFile_meta: true,
+  async renameFile({ folder, file, newFile, fileType }) {
+    await fs.rename(
+      path.join(path.join(appdir(), folder), `${file}.${fileType}`),
+      path.join(path.join(appdir(), folder), `${newFile}.${fileType}`)
+    );
+    socket.emitChanged(`app-files-changed-${folder}`);
+    this.emitChangedDbApp(folder);
+  },
+
+  renameFolder_meta: true,
+  async renameFolder({ folder, newFolder }) {
+    const uniqueName = await this.getNewAppFolder({ name: newFolder });
+    await fs.rename(path.join(appdir(), folder), path.join(appdir(), uniqueName));
+    socket.emitChanged(`app-folders-changed`);
+  },
+
+  deleteFolder_meta: true,
+  async deleteFolder({ folder }) {
+    if (!folder) throw new Error('Missing folder parameter');
+    await fs.rmdir(path.join(appdir(), folder), { recursive: true });
+    socket.emitChanged(`app-folders-changed`);
+    socket.emitChanged(`app-files-changed-${folder}`);
+    socket.emitChanged('used-apps-changed');
+  },
+
+  async getNewAppFolder({ name }) {
+    if (!(await fs.exists(path.join(appdir(), name)))) return name;
+    let index = 2;
+    while (await fs.exists(path.join(appdir(), `${name}${index}`))) {
+      index += 1;
+    }
+    return `${name}${index}`;
+  },
+
+  getUsedAppFolders_meta: true,
+  async getUsedAppFolders() {
+    const list = await connections.list();
+    const apps = [];
+
+    for (const connection of list) {
+      for (const db of connection.databases || []) {
+        for (const key of _.keys(db || {})) {
+          if (key.startsWith('useApp:') && db[key]) {
+            apps.push(key.substring('useApp:'.length));
+          }
+        }
+      }
+    }
+
+    return _.intersection(_.uniq(apps), await fs.readdir(appdir()));
+  },
+
+  getUsedApps_meta: true,
+  async getUsedApps() {
+    const apps = await this.getUsedAppFolders();
+    const res = [];
+
+    for (const folder of apps) {
+      res.push(await this.loadApp({ folder }));
+    }
+    return res;
+  },
+
+  // getAppsForDb_meta: true,
+  // async getAppsForDb({ conid, database }) {
+  //   const connection = await connections.get({ conid });
+  //   if (!connection) return [];
+  //   const db = (connection.databases || []).find(x => x.name == database);
+  //   const apps = [];
+  //   const res = [];
+  //   if (db) {
+  //     for (const key of _.keys(db || {})) {
+  //       if (key.startsWith('useApp:') && db[key]) {
+  //         apps.push(key.substring('useApp:'.length));
+  //       }
+  //     }
+  //   }
+  //   for (const folder of apps) {
+  //     res.push(await this.loadApp({ folder }));
+  //   }
+  //   return res;
+  // },
+
+  loadApp_meta: true,
+  async loadApp({ folder }) {
+    const res = {
+      queries: [],
+      commands: [],
+      name: folder,
+    };
+    const dir = path.join(appdir(), folder);
+    if (await fs.exists(dir)) {
+      const files = await fs.readdir(dir);
+
+      async function processType(ext, field) {
+        for (const file of files) {
+          if (file.endsWith(ext)) {
+            res[field].push({
+              name: file.slice(0, -ext.length),
+              sql: await fs.readFile(path.join(dir, file), { encoding: 'utf-8' }),
+            });
+          }
+        }
+      }
+
+      await processType('.command.sql', 'commands');
+      await processType('.query.sql', 'queries');
+    }
+
+    try {
+      res.virtualReferences = JSON.parse(
+        await fs.readFile(path.join(dir, 'virtual-references.config.json'), { encoding: 'utf-8' })
+      );
+    } catch (err) {
+      res.virtualReferences = [];
+    }
+    try {
+      res.dictionaryDescriptions = JSON.parse(
+        await fs.readFile(path.join(dir, 'dictionary-descriptions.config.json'), { encoding: 'utf-8' })
+      );
+    } catch (err) {
+      res.dictionaryDescriptions = [];
+    }
+
+    return res;
+  },
+
+  async saveConfigFile(appFolder, filename, filterFunc, newItem) {
+    const file = path.join(appdir(), appFolder, filename);
+
+    let json;
+    try {
+      json = JSON.parse(await fs.readFile(file, { encoding: 'utf-8' }));
+    } catch (err) {
+      json = [];
+    }
+
+    if (filterFunc) {
+      json = json.filter(filterFunc);
+    }
+
+    json = [...json, newItem];
+
+    await fs.writeFile(file, JSON.stringify(json, undefined, 2));
+
+    socket.emitChanged(`app-files-changed-${appFolder}`);
+    socket.emitChanged('used-apps-changed');
+  },
+
+  saveVirtualReference_meta: true,
+  async saveVirtualReference({ appFolder, schemaName, pureName, refSchemaName, refTableName, columns }) {
+    await this.saveConfigFile(
+      appFolder,
+      'virtual-references.config.json',
+      columns.length == 1
+        ? x =>
+            !(
+              x.schemaName == schemaName &&
+              x.pureName == pureName &&
+              x.columns.length == 1 &&
+              x.columns[0].columnName == columns[0].columnName
+            )
+        : null,
+      {
+        schemaName,
+        pureName,
+        refSchemaName,
+        refTableName,
+        columns,
+      }
+    );
+    return true;
+  },
+
+  saveDictionaryDescription_meta: true,
+  async saveDictionaryDescription({ appFolder, pureName, schemaName, expression, columns, delimiter }) {
+    await this.saveConfigFile(
+      appFolder,
+      'dictionary-descriptions.config.json',
+      x => !(x.schemaName == schemaName && x.pureName == pureName),
+      {
+        schemaName,
+        pureName,
+        expression,
+        columns,
+        delimiter,
+      }
+    );
+
+    return true;
+  },
+
+  createConfigFile_meta: true,
+  async createConfigFile({ appFolder, fileName, content }) {
+    const file = path.join(appdir(), appFolder, fileName);
+    if (!(await fs.exists(file))) {
+      await fs.writeFile(file, JSON.stringify(content, undefined, 2));
+      socket.emitChanged(`app-files-changed-${appFolder}`);
+      socket.emitChanged('used-apps-changed');
+      return true;
+    }
+    return false;
+  },
+};

packages/api/src/controllers/archive.js

@@ -3,13 +3,14 @@ const stream = require('stream');
 const readline = require('readline');
 const path = require('path');
 const { formatWithOptions } = require('util');
-const { archivedir } = require('../utility/directories');
+const { archivedir, clearArchiveLinksCache, resolveArchiveFolder } = require('../utility/directories');
 const socket = require('../utility/socket');
 const JsonLinesDatastore = require('../utility/JsonLinesDatastore');
 const { saveFreeTableData } = require('../utility/freeTableStorage');
+const loadFilesRecursive = require('../utility/loadFilesRecursive');
 
 module.exports = {
-  folders_meta: 'get',
+  folders_meta: true,
   async folders() {
     const folders = await fs.readdir(archivedir());
     return [
@@ -26,59 +27,103 @@ module.exports = {
     ];
   },
 
-  createFolder_meta: 'post',
+  createFolder_meta: true,
   async createFolder({ folder }) {
     await fs.mkdir(path.join(archivedir(), folder));
     socket.emitChanged('archive-folders-changed');
     return true;
   },
 
-  files_meta: 'get',
-  async files({ folder }) {
-    const dir = path.join(archivedir(), folder);
-    if (!(await fs.exists(dir))) return [];
-    const files = await fs.readdir(dir);
-    return files
-      .filter(name => name.endsWith('.jsonl'))
-      .map(name => ({
-        name: name.slice(0, -'.jsonl'.length),
-        type: 'jsonl',
-      }));
+  createLink_meta: true,
+  async createLink({ linkedFolder }) {
+    const folder = await this.getNewArchiveFolder({ database: path.parse(linkedFolder).name + '.link' });
+    fs.writeFile(path.join(archivedir(), folder), linkedFolder);
+    clearArchiveLinksCache();
+    socket.emitChanged('archive-folders-changed');
+    return folder;
   },
 
-  refreshFiles_meta: 'post',
+  files_meta: true,
+  async files({ folder }) {
+    const dir = resolveArchiveFolder(folder);
+    if (!(await fs.exists(dir))) return [];
+    const files = await loadFilesRecursive(dir); // fs.readdir(dir);
+
+    function fileType(ext, type) {
+      return files
+        .filter(name => name.endsWith(ext))
+        .map(name => ({
+          name: name.slice(0, -ext.length),
+          label: path.parse(name.slice(0, -ext.length)).base,
+          type,
+        }));
+    }
+
+    return [
+      ...fileType('.jsonl', 'jsonl'),
+      ...fileType('.table.yaml', 'table.yaml'),
+      ...fileType('.view.sql', 'view.sql'),
+      ...fileType('.proc.sql', 'proc.sql'),
+      ...fileType('.func.sql', 'func.sql'),
+      ...fileType('.trigger.sql', 'trigger.sql'),
+      ...fileType('.matview.sql', 'matview.sql'),
+    ];
+  },
+
+  refreshFiles_meta: true,
   async refreshFiles({ folder }) {
     socket.emitChanged(`archive-files-changed-${folder}`);
   },
 
-  refreshFolders_meta: 'post',
+  refreshFolders_meta: true,
   async refreshFolders() {
     socket.emitChanged(`archive-folders-changed`);
   },
 
-  deleteFile_meta: 'post',
-  async deleteFile({ folder, file }) {
-    await fs.unlink(path.join(archivedir(), folder, `${file}.jsonl`));
+  deleteFile_meta: true,
+  async deleteFile({ folder, file, fileType }) {
+    await fs.unlink(path.join(resolveArchiveFolder(folder), `${file}.${fileType}`));
     socket.emitChanged(`archive-files-changed-${folder}`);
   },
 
-  deleteFolder_meta: 'post',
-  async deleteFolder({ folder }) {
-    if (!folder) throw new Error('Missing folder parameter');
-    await fs.rmdir(path.join(archivedir(), folder), { recursive: true });
+  renameFile_meta: true,
+  async renameFile({ folder, file, newFile, fileType }) {
+    await fs.rename(
+      path.join(resolveArchiveFolder(folder), `${file}.${fileType}`),
+      path.join(resolveArchiveFolder(folder), `${newFile}.${fileType}`)
+    );
+    socket.emitChanged(`archive-files-changed-${folder}`);
+  },
+
+  renameFolder_meta: true,
+  async renameFolder({ folder, newFolder }) {
+    const uniqueName = await this.getNewArchiveFolder({ database: newFolder });
+    await fs.rename(path.join(archivedir(), folder), path.join(archivedir(), uniqueName));
     socket.emitChanged(`archive-folders-changed`);
   },
 
-  saveFreeTable_meta: 'post',
+  deleteFolder_meta: true,
+  async deleteFolder({ folder }) {
+    if (!folder) throw new Error('Missing folder parameter');
+    if (folder.endsWith('.link')) {
+      await fs.unlink(path.join(archivedir(), folder));
+    } else {
+      await fs.rmdir(path.join(archivedir(), folder), { recursive: true });
+    }
+    socket.emitChanged(`archive-folders-changed`);
+  },
+
+  saveFreeTable_meta: true,
   async saveFreeTable({ folder, file, data }) {
-    saveFreeTableData(path.join(archivedir(), folder, `${file}.jsonl`), data);
+    await saveFreeTableData(path.join(resolveArchiveFolder(folder), `${file}.jsonl`), data);
+    socket.emitChanged(`archive-files-changed-${folder}`);
     return true;
   },
 
-  loadFreeTable_meta: 'post',
+  loadFreeTable_meta: true,
   async loadFreeTable({ folder, file }) {
     return new Promise((resolve, reject) => {
-      const fileStream = fs.createReadStream(path.join(archivedir(), folder, `${file}.jsonl`));
+      const fileStream = fs.createReadStream(path.join(resolveArchiveFolder(folder), `${file}.jsonl`));
       const liner = readline.createInterface({
         input: fileStream,
       });
@@ -95,4 +140,23 @@ module.exports = {
       });
     });
   },
+
+  saveText_meta: true,
+  async saveText({ folder, file, text }) {
+    await fs.writeFile(path.join(resolveArchiveFolder(folder), `${file}.jsonl`), text);
+    socket.emitChanged(`archive-files-changed-${folder}`);
+    return true;
+  },
+
+  async getNewArchiveFolder({ database }) {
+    const isLink = database.endsWith(database);
+    const name = isLink ? database.slice(0, -5) : database;
+    const suffix = isLink ? '.link' : '';
+    if (!(await fs.exists(path.join(archivedir(), database)))) return database;
+    let index = 2;
+    while (await fs.exists(path.join(archivedir(), `${name}${index}${suffix}`))) {
+      index += 1;
+    }
+    return `${name}${index}${suffix}`;
+  },
 };

packages/api/src/controllers/config.js

@@ -1,61 +1,109 @@
 const fs = require('fs-extra');
+const os = require('os');
 const path = require('path');
+const axios = require('axios');
 const { datadir } = require('../utility/directories');
-const hasPermission = require('../utility/hasPermission');
+const { hasPermission, getLogins } = require('../utility/hasPermission');
 const socket = require('../utility/socket');
 const _ = require('lodash');
+const AsyncLock = require('async-lock');
 
 const currentVersion = require('../currentVersion');
 const platformInfo = require('../utility/platformInfo');
 const connections = require('../controllers/connections');
 
+const lock = new AsyncLock();
+
 module.exports = {
-  settingsValue: {},
+  // settingsValue: {},
 
-  async _init() {
-    try {
-      this.settingsValue = JSON.parse(await fs.readFile(path.join(datadir(), 'settings.json'), { encoding: 'utf-8' }));
-    } catch (err) {
-      this.settingsValue = {};
-    }
-  },
+  // async _init() {
+  //   try {
+  //     this.settingsValue = JSON.parse(await fs.readFile(path.join(datadir(), 'settings.json'), { encoding: 'utf-8' }));
+  //   } catch (err) {
+  //     this.settingsValue = {};
+  //   }
+  // },
 
-  get_meta: 'get',
-  async get() {
-    const permissions = process.env.PERMISSIONS ? process.env.PERMISSIONS.split(',') : null;
+  get_meta: true,
+  async get(_params, req) {
+    const logins = getLogins();
+    const login = logins ? logins.find(x => x.login == (req.auth && req.auth.user)) : null;
+    const permissions = login ? login.permissions : null;
 
     return {
       runAsPortal: !!connections.portalConnections,
       singleDatabase: connections.singleDatabase,
+      // hideAppEditor: !!process.env.HIDE_APP_EDITOR,
+      allowShellConnection: platformInfo.allowShellConnection,
+      allowShellScripting: platformInfo.allowShellConnection,
+      isDocker: platformInfo.isDocker,
       permissions,
+      login,
       ...currentVersion,
     };
   },
 
-  platformInfo_meta: 'get',
+  logout_meta: {
+    method: 'get',
+    raw: true,
+  },
+  logout(req, res) {
+    res.status(401).send('Logged out<br><a href="../..">Back to DbGate</a>');
+  },
+
+  platformInfo_meta: true,
   async platformInfo() {
     return platformInfo;
   },
 
-  getSettings_meta: 'get',
+  getSettings_meta: true,
   async getSettings() {
-    return this.settingsValue;
-  },
-
-  updateSettings_meta: 'post',
-  async updateSettings(values) {
-    if (!hasPermission(`settings/change`)) return false;
     try {
-      const updated = {
-        ...this.settingsValue,
-        ...values,
-      };
-      await fs.writeFile(path.join(datadir(), 'settings.json'), JSON.stringify(updated, undefined, 2));
-      this.settingsValue = updated;
-      socket.emitChanged(`settings-changed`);
-      return updated;
+      return this.fillMissingSettings(
+        JSON.parse(await fs.readFile(path.join(datadir(), 'settings.json'), { encoding: 'utf-8' }))
+      );
     } catch (err) {
-      return false;
+      return this.fillMissingSettings({});
     }
   },
+
+  fillMissingSettings(value) {
+    const res = {
+      ...value,
+    };
+    if (value['app.useNativeMenu'] !== true && value['app.useNativeMenu'] !== false) {
+      // res['app.useNativeMenu'] = os.platform() == 'darwin' ? true : false;
+      res['app.useNativeMenu'] = false;
+    }
+    return res;
+  },
+
+  updateSettings_meta: true,
+  async updateSettings(values, req) {
+    if (!hasPermission(`settings/change`, req)) return false;
+
+    const res = await lock.acquire('update', async () => {
+      const currentValue = await this.getSettings();
+      try {
+        const updated = {
+          ...currentValue,
+          ...values,
+        };
+        await fs.writeFile(path.join(datadir(), 'settings.json'), JSON.stringify(updated, undefined, 2));
+        // this.settingsValue = updated;
+        socket.emitChanged(`settings-changed`);
+        return updated;
+      } catch (err) {
+        return false;
+      }
+    });
+    return res;
+  },
+
+  changelog_meta: true,
+  async changelog() {
+    const resp = await axios.default.get('https://raw.githubusercontent.com/dbgate/dbgate/master/CHANGELOG.md');
+    return resp.data;
+  },
 };

packages/api/src/controllers/connections.js

@@ -1,12 +1,18 @@
 const path = require('path');
 const { fork } = require('child_process');
 const _ = require('lodash');
-const nedb = require('nedb-promises');
+const fs = require('fs-extra');
 
-const { datadir } = require('../utility/directories');
+const { datadir, filesdir } = require('../utility/directories');
 const socket = require('../utility/socket');
-const { encryptConnection } = require('../utility/crypting');
+const { encryptConnection, maskConnection } = require('../utility/crypting');
 const { handleProcessCommunication } = require('../utility/processComm');
+const { pickSafeConnectionInfo } = require('../utility/crypting');
+const JsonLinesDatabase = require('../utility/JsonLinesDatabase');
+
+const processArgs = require('../utility/processArgs');
+const { safeJsonParse } = require('dbgate-tools');
+const platformInfo = require('../utility/platformInfo');
 
 function getNamedArgs() {
   const res = {};
@@ -36,7 +42,7 @@ function getDatabaseFileLabel(databaseFile) {
 
 function getPortalCollections() {
   if (process.env.CONNECTIONS) {
-    return _.compact(process.env.CONNECTIONS.split(',')).map(id => ({
+    const connections = _.compact(process.env.CONNECTIONS.split(',')).map(id => ({
       _id: id,
       engine: process.env[`ENGINE_${id}`],
       server: process.env[`SERVER_${id}`],
@@ -44,11 +50,44 @@ function getPortalCollections() {
       password: process.env[`PASSWORD_${id}`],
       port: process.env[`PORT_${id}`],
       databaseUrl: process.env[`URL_${id}`],
+      useDatabaseUrl: !!process.env[`URL_${id}`],
       databaseFile: process.env[`FILE_${id}`],
-      defaultDatabase: process.env[`DATABASE_${id}`],
-      singleDatabase: !!process.env[`DATABASE_${id}`],
+      defaultDatabase:
+        process.env[`DATABASE_${id}`] ||
+        (process.env[`FILE_${id}`] ? getDatabaseFileLabel(process.env[`FILE_${id}`]) : null),
+      singleDatabase: !!process.env[`DATABASE_${id}`] || !!process.env[`FILE_${id}`],
       displayName: process.env[`LABEL_${id}`],
+      isReadOnly: process.env[`READONLY_${id}`],
+      databases: process.env[`DBCONFIG_${id}`] ? safeJsonParse(process.env[`DBCONFIG_${id}`]) : null,
+
+      // SSH tunnel
+      useSshTunnel: process.env[`USE_SSH_${id}`],
+      sshHost: process.env[`SSH_HOST_${id}`],
+      sshPort: process.env[`SSH_PORT_${id}`],
+      sshMode: process.env[`SSH_MODE_${id}`],
+      sshLogin: process.env[`SSH_LOGIN_${id}`],
+      sshPassword: process.env[`SSH_PASSWORD_${id}`],
+      sshKeyfile: process.env[`SSH_KEY_FILE_${id}`],
+      sshKeyfilePassword: process.env[`SSH_KEY_FILE_PASSWORD_${id}`],
+
+      // SSL
+      useSsl: process.env[`USE_SSL_${id}`],
+      sslCaFile: process.env[`SSL_CA_FILE_${id}`],
+      sslCertFile: process.env[`SSL_CERT_FILE_${id}`],
+      sslCertFilePassword: process.env[`SSL_CERT_FILE_PASSWORD_${id}`],
+      sslKeyFile: process.env[`SSL_KEY_FILE_${id}`],
+      sslRejectUnauthorized: process.env[`SSL_REJECT_UNAUTHORIZED_${id}`],
     }));
+    console.log('Using connections from ENV variables:');
+    console.log(JSON.stringify(connections.map(pickSafeConnectionInfo), undefined, 2));
+    const noengine = connections.filter(x => !x.engine);
+    if (noengine.length > 0) {
+      console.log(
+        'Warning: Invalid CONNECTIONS configutation, missing ENGINE for connection ID:',
+        noengine.map(x => x._id)
+      );
+    }
+    return connections;
   }
 
   const args = getNamedArgs();
@@ -121,58 +160,121 @@ module.exports = {
     const dir = datadir();
     if (!portalConnections) {
       // @ts-ignore
-      this.datastore = nedb.create(path.join(dir, 'connections.jsonl'));
+      this.datastore = new JsonLinesDatabase(path.join(dir, 'connections.jsonl'));
     }
   },
 
-  list_meta: 'get',
+  list_meta: true,
   async list() {
-    return portalConnections || this.datastore.find();
+    return portalConnections && !platformInfo.allowShellConnection
+      ? portalConnections.map(maskConnection)
+      : this.datastore.find();
   },
 
-  test_meta: {
-    method: 'post',
-    raw: true,
-  },
-  test(req, res) {
-    const subprocess = fork(process.argv[1], ['--start-process', 'connectProcess', ...process.argv.slice(3)]);
-    subprocess.on('message', resp => {
-      if (handleProcessCommunication(resp, subprocess)) return;
-      // @ts-ignore
-      const { msgtype } = resp;
-      if (msgtype == 'connected' || msgtype == 'error') {
-        res.json(resp);
-      }
+  test_meta: true,
+  test(connection) {
+    const subprocess = fork(global['API_PACKAGE'] || process.argv[1], [
+      '--is-forked-api',
+      '--start-process',
+      'connectProcess',
+      ...processArgs.getPassArgs(),
+      // ...process.argv.slice(3),
+    ]);
+    subprocess.send(connection);
+    return new Promise(resolve => {
+      subprocess.on('message', resp => {
+        if (handleProcessCommunication(resp, subprocess)) return;
+        // @ts-ignore
+        const { msgtype } = resp;
+        if (msgtype == 'connected' || msgtype == 'error') {
+          resolve(resp);
+        }
+      });
     });
-    subprocess.send(req.body);
   },
 
-  save_meta: 'post',
+  save_meta: true,
   async save(connection) {
     if (portalConnections) return;
     let res;
     const encrypted = encryptConnection(connection);
     if (connection._id) {
-      res = await this.datastore.update(_.pick(connection, '_id'), encrypted);
+      res = await this.datastore.update(encrypted);
     } else {
       res = await this.datastore.insert(encrypted);
     }
     socket.emitChanged('connection-list-changed');
+    socket.emitChanged('used-apps-changed');
+    if (this._closeAll) {
+      this._closeAll(connection._id);
+    }
+    // for (const db of connection.databases || []) {
+    //   socket.emitChanged(`db-apps-changed-${connection._id}-${db.name}`);
+    // }
     return res;
   },
 
-  delete_meta: 'post',
-  async delete(connection) {
+  update_meta: true,
+  async update({ _id, values }) {
     if (portalConnections) return;
-    const res = await this.datastore.remove(_.pick(connection, '_id'));
+    const res = await this.datastore.patch(_id, values);
     socket.emitChanged('connection-list-changed');
     return res;
   },
 
-  get_meta: 'get',
+  updateDatabase_meta: true,
+  async updateDatabase({ conid, database, values }) {
+    if (portalConnections) return;
+    const conn = await this.datastore.get(conid);
+    let databases = (conn && conn.databases) || [];
+    if (databases.find(x => x.name == database)) {
+      databases = databases.map(x => (x.name == database ? { ...x, ...values } : x));
+    } else {
+      databases = [...databases, { name: database, ...values }];
+    }
+    const res = await this.datastore.patch(conid, { databases });
+    socket.emitChanged('connection-list-changed');
+    socket.emitChanged('used-apps-changed');
+    // socket.emitChanged(`db-apps-changed-${conid}-${database}`);
+    return res;
+  },
+
+  delete_meta: true,
+  async delete(connection) {
+    if (portalConnections) return;
+    const res = await this.datastore.remove(connection._id);
+    socket.emitChanged('connection-list-changed');
+    return res;
+  },
+
+  async getCore({ conid, mask = false }) {
+    if (!conid) return null;
+    if (portalConnections) {
+      const res = portalConnections.find(x => x._id == conid) || null;
+      return mask && !platformInfo.allowShellConnection ? maskConnection(res) : res;
+    }
+    const res = await this.datastore.get(conid);
+    return res || null;
+  },
+
+  get_meta: true,
   async get({ conid }) {
-    if (portalConnections) return portalConnections.find(x => x._id == conid);
-    const res = await this.datastore.find({ _id: conid });
-    return res[0];
+    return this.getCore({ conid, mask: true });
+  },
+
+  newSqliteDatabase_meta: true,
+  async newSqliteDatabase({ file }) {
+    const sqliteDir = path.join(filesdir(), 'sqlite');
+    if (!(await fs.exists(sqliteDir))) {
+      await fs.mkdir(sqliteDir);
+    }
+    const databaseFile = path.join(sqliteDir, `${file}.sqlite`);
+    const res = await this.save({
+      engine: 'sqlite@dbgate-plugin-sqlite',
+      databaseFile,
+      singleDatabase: true,
+      defaultDatabase: `${file}.sqlite`,
+    });
+    return res;
   },
 };

packages/api/src/controllers/databaseConnections.js

@@ -1,10 +1,31 @@
 const uuidv1 = require('uuid/v1');
 const connections = require('./connections');
+const archive = require('./archive');
 const socket = require('../utility/socket');
 const { fork } = require('child_process');
-const { DatabaseAnalyser } = require('dbgate-tools');
+const {
+  DatabaseAnalyser,
+  computeDbDiffRows,
+  getCreateObjectScript,
+  getAlterDatabaseScript,
+  generateDbPairingId,
+  matchPairedObjects,
+  extendDatabaseInfo,
+  modelCompareDbDiffOptions,
+} = require('dbgate-tools');
+const { html, parse } = require('diff2html');
 const { handleProcessCommunication } = require('../utility/processComm');
 const config = require('./config');
+const fs = require('fs-extra');
+const exportDbModel = require('../utility/exportDbModel');
+const { archivedir, resolveArchiveFolder, uploadsdir } = require('../utility/directories');
+const path = require('path');
+const importDbModel = require('../utility/importDbModel');
+const requireEngineDriver = require('../utility/requireEngineDriver');
+const generateDeploySql = require('../shell/generateDeploySql');
+const { createTwoFilesPatch } = require('diff');
+const diff2htmlPage = require('../utility/diff2htmlPage');
+const processArgs = require('../utility/processArgs');
 
 module.exports = {
   /** @type {import('dbgate-types').OpenedDatabaseConnection[]} */
@@ -12,6 +33,10 @@ module.exports = {
   closed: {},
   requests: {},
 
+  async _init() {
+    connections._closeAll = conid => this.closeAll(conid);
+  },
+
   handle_structure(conid, database, { structure }) {
     const existing = this.opened.find(x => x.conid == conid && x.database == database);
     if (!existing) return;
@@ -41,9 +66,10 @@ module.exports = {
     delete this.requests[msgid];
   },
   handle_status(conid, database, { status }) {
+    // console.log('HANDLE SET STATUS', status);
     const existing = this.opened.find(x => x.conid == conid && x.database == database);
     if (!existing) return;
-    if (existing.status == status) return;
+    if (existing.status && status && existing.status.counter > status.counter) return;
     existing.status = status;
     socket.emitChanged(`database-status-changed-${conid}-${database}`);
   },
@@ -53,11 +79,13 @@ module.exports = {
   async ensureOpened(conid, database) {
     const existing = this.opened.find(x => x.conid == conid && x.database == database);
     if (existing) return existing;
-    const connection = await connections.get({ conid });
-    const subprocess = fork(process.argv[1], [
+    const connection = await connections.getCore({ conid });
+    const subprocess = fork(global['API_PACKAGE'] || process.argv[1], [
+      '--is-forked-api',
       '--start-process',
       'databaseConnectionProcess',
-      ...process.argv.slice(3),
+      ...processArgs.getPassArgs(),
+      // ...process.argv.slice(3),
     ]);
     const lastClosed = this.closed[`${conid}/${database}`];
     const newOpened = {
@@ -86,7 +114,7 @@ module.exports = {
       msgtype: 'connect',
       connection: { ...connection, database },
       structure: lastClosed ? lastClosed.structure : null,
-      globalSettings: config.settingsValue,
+      globalSettings: await config.getSettings(),
     });
     return newOpened;
   },
@@ -101,7 +129,7 @@ module.exports = {
     return promise;
   },
 
-  queryData_meta: 'post',
+  queryData_meta: true,
   async queryData({ conid, database, sql }) {
     console.log(`Processing query, conid=${conid}, database=${database}, sql=${sql}`);
     const opened = await this.ensureOpened(conid, database);
@@ -112,7 +140,14 @@ module.exports = {
     return res;
   },
 
-  runScript_meta: 'post',
+  sqlSelect_meta: true,
+  async sqlSelect({ conid, database, select }) {
+    const opened = await this.ensureOpened(conid, database);
+    const res = await this.sendRequest(opened, { msgtype: 'sqlSelect', select });
+    return res;
+  },
+
+  runScript_meta: true,
   async runScript({ conid, database, sql }) {
     console.log(`Processing script, conid=${conid}, database=${database}, sql=${sql}`);
     const opened = await this.ensureOpened(conid, database);
@@ -120,21 +155,76 @@ module.exports = {
     return res;
   },
 
-  collectionData_meta: 'post',
+  collectionData_meta: true,
   async collectionData({ conid, database, options }) {
     const opened = await this.ensureOpened(conid, database);
     const res = await this.sendRequest(opened, { msgtype: 'collectionData', options });
-    return res.result;
+    return res.result || null;
   },
 
-  updateCollection_meta: 'post',
+  async loadDataCore(msgtype, { conid, database, ...args }) {
+    const opened = await this.ensureOpened(conid, database);
+    const res = await this.sendRequest(opened, { msgtype, ...args });
+    if (res.errorMessage) {
+      console.error(res.errorMessage);
+
+      return {
+        errorMessage: res.errorMessage,
+      };
+    }
+    return res.result || null;
+  },
+
+  loadKeys_meta: true,
+  async loadKeys({ conid, database, root, filter }) {
+    return this.loadDataCore('loadKeys', { conid, database, root, filter });
+  },
+
+  exportKeys_meta: true,
+  async exportKeys({ conid, database, options }) {
+    return this.loadDataCore('exportKeys', { conid, database, options });
+  },
+
+  loadKeyInfo_meta: true,
+  async loadKeyInfo({ conid, database, key }) {
+    return this.loadDataCore('loadKeyInfo', { conid, database, key });
+  },
+
+  loadKeyTableRange_meta: true,
+  async loadKeyTableRange({ conid, database, key, cursor, count }) {
+    return this.loadDataCore('loadKeyTableRange', { conid, database, key, cursor, count });
+  },
+
+  loadFieldValues_meta: true,
+  async loadFieldValues({ conid, database, schemaName, pureName, field, search }) {
+    return this.loadDataCore('loadFieldValues', { conid, database, schemaName, pureName, field, search });
+  },
+
+  callMethod_meta: true,
+  async callMethod({ conid, database, method, args }) {
+    return this.loadDataCore('callMethod', { conid, database, method, args });
+
+    // const opened = await this.ensureOpened(conid, database);
+    // const res = await this.sendRequest(opened, { msgtype: 'callMethod', method, args });
+    // if (res.errorMessage) {
+    //   console.error(res.errorMessage);
+    // }
+    // return res.result || null;
+  },
+
+  updateCollection_meta: true,
   async updateCollection({ conid, database, changeSet }) {
     const opened = await this.ensureOpened(conid, database);
     const res = await this.sendRequest(opened, { msgtype: 'updateCollection', changeSet });
-    return res.result;
+    if (res.errorMessage) {
+      return {
+        errorMessage: res.errorMessage,
+      };
+    }
+    return res.result || null;
   },
 
-  status_meta: 'get',
+  status_meta: true,
   async status({ conid, database }) {
     const existing = this.opened.find(x => x.conid == conid && x.database == database);
     if (existing) {
@@ -156,7 +246,7 @@ module.exports = {
     };
   },
 
-  ping_meta: 'post',
+  ping_meta: true,
   async ping({ conid, database }) {
     let existing = this.opened.find(x => x.conid == conid && x.database == database);
 
@@ -172,7 +262,7 @@ module.exports = {
     };
   },
 
-  refresh_meta: 'post',
+  refresh_meta: true,
   async refresh({ conid, database, keepOpen }) {
     if (!keepOpen) this.close(conid, database);
 
@@ -180,10 +270,10 @@ module.exports = {
     return { status: 'ok' };
   },
 
-  syncModel_meta: 'post',
-  async syncModel({ conid, database }) {
+  syncModel_meta: true,
+  async syncModel({ conid, database, isFullRefresh }) {
     const conn = await this.ensureOpened(conid, database);
-    conn.subprocess.send({ msgtype: 'syncModel' });
+    conn.subprocess.send({ msgtype: 'syncModel', isFullRefresh });
     return { status: 'ok' };
   },
 
@@ -204,14 +294,25 @@ module.exports = {
     }
   },
 
-  disconnect_meta: 'post',
+  closeAll(conid, kill = true) {
+    for (const existing of this.opened.filter(x => x.conid == conid)) {
+      this.close(conid, existing.database, kill);
+    }
+  },
+
+  disconnect_meta: true,
   async disconnect({ conid, database }) {
     await this.close(conid, database, true);
     return { status: 'ok' };
   },
 
-  structure_meta: 'get',
+  structure_meta: true,
   async structure({ conid, database }) {
+    if (conid == '__model') {
+      const model = await importDbModel(database);
+      return model;
+    }
+
     const opened = await this.ensureOpened(conid, database);
     return opened.structure;
     // const existing = this.opened.find((x) => x.conid == conid && x.database == database);
@@ -222,13 +323,14 @@ module.exports = {
     // };
   },
 
-  serverVersion_meta: 'get',
+  serverVersion_meta: true,
   async serverVersion({ conid, database }) {
+    if (!conid) return null;
     const opened = await this.ensureOpened(conid, database);
-    return opened.serverVersion;
+    return opened.serverVersion || null;
   },
 
-  sqlPreview_meta: 'post',
+  sqlPreview_meta: true,
   async sqlPreview({ conid, database, objects, options }) {
     // wait for structure
     await this.structure({ conid, database });
@@ -238,11 +340,99 @@ module.exports = {
     return res;
   },
 
-  // runCommand_meta: 'post',
+  exportModel_meta: true,
+  async exportModel({ conid, database }) {
+    const archiveFolder = await archive.getNewArchiveFolder({ database });
+    await fs.mkdir(path.join(archivedir(), archiveFolder));
+    const model = await this.structure({ conid, database });
+    await exportDbModel(model, path.join(archivedir(), archiveFolder));
+    socket.emitChanged(`archive-folders-changed`);
+    return { archiveFolder };
+  },
+
+  generateDeploySql_meta: true,
+  async generateDeploySql({ conid, database, archiveFolder }) {
+    const opened = await this.ensureOpened(conid, database);
+    const res = await this.sendRequest(opened, {
+      msgtype: 'generateDeploySql',
+      modelFolder: resolveArchiveFolder(archiveFolder),
+    });
+    return res;
+
+    // const connection = await connections.get({ conid });
+    // return generateDeploySql({
+    //   connection,
+    //   analysedStructure: await this.structure({ conid, database }),
+    //   modelFolder: resolveArchiveFolder(archiveFolder),
+    // });
+
+    // const deployedModel = generateDbPairingId(await importDbModel(path.join(archivedir(), archiveFolder)));
+    // const currentModel = generateDbPairingId(await this.structure({ conid, database }));
+    // const currentModelPaired = matchPairedObjects(deployedModel, currentModel);
+    // const connection = await connections.get({ conid });
+    // const driver = requireEngineDriver(connection);
+    // const { sql } = getAlterDatabaseScript(currentModelPaired, deployedModel, {}, deployedModel, driver);
+    // return {
+    //   deployedModel,
+    //   currentModel,
+    //   currentModelPaired,
+    //   sql,
+    // };
+    // return sql;
+  },
+  // runCommand_meta: true,
   // async runCommand({ conid, database, sql }) {
   //   console.log(`Running SQL command , conid=${conid}, database=${database}, sql=${sql}`);
   //   const opened = await this.ensureOpened(conid, database);
   //   const res = await this.sendRequest(opened, { msgtype: 'queryData', sql });
   //   return res;
   // },
+
+  async getUnifiedDiff({ sourceConid, sourceDatabase, targetConid, targetDatabase }) {
+    const dbDiffOptions = sourceConid == '__model' ? modelCompareDbDiffOptions : {};
+
+    const sourceDb = generateDbPairingId(
+      extendDatabaseInfo(await this.structure({ conid: sourceConid, database: sourceDatabase }))
+    );
+    const targetDb = generateDbPairingId(
+      extendDatabaseInfo(await this.structure({ conid: targetConid, database: targetDatabase }))
+    );
+    // const sourceConnection = await connections.getCore({conid:sourceConid})
+    const connection = await connections.getCore({ conid: targetConid });
+    const driver = requireEngineDriver(connection);
+    const targetDbPaired = matchPairedObjects(sourceDb, targetDb, dbDiffOptions);
+    const diffRows = computeDbDiffRows(sourceDb, targetDbPaired, dbDiffOptions, driver);
+
+    // console.log('sourceDb', sourceDb);
+    // console.log('targetDb', targetDb);
+    // console.log('sourceConid, sourceDatabase', sourceConid, sourceDatabase);
+
+    let res = '';
+    for (const row of diffRows) {
+      // console.log('PAIR', row.source && row.source.pureName, row.target && row.target.pureName);
+      const unifiedDiff = createTwoFilesPatch(
+        (row.target && row.target.pureName) || '',
+        (row.source && row.source.pureName) || '',
+        getCreateObjectScript(row.target, driver),
+        getCreateObjectScript(row.source, driver),
+        '',
+        ''
+      );
+      res += unifiedDiff;
+    }
+    return res;
+  },
+
+  generateDbDiffReport_meta: true,
+  async generateDbDiffReport({ filePath, sourceConid, sourceDatabase, targetConid, targetDatabase }) {
+    const unifiedDiff = await this.getUnifiedDiff({ sourceConid, sourceDatabase, targetConid, targetDatabase });
+
+    const diffJson = parse(unifiedDiff);
+    // $: diffHtml = html(diffJson, { outputFormat: 'side-by-side', drawFileList: false });
+    const diffHtml = html(diffJson, { outputFormat: 'side-by-side' });
+
+    await fs.writeFile(filePath, diff2htmlPage(diffHtml));
+
+    return true;
+  },
 };

packages/api/src/controllers/files.js

@@ -1,9 +1,13 @@
+const uuidv1 = require('uuid/v1');
 const fs = require('fs-extra');
 const path = require('path');
-const { filesdir } = require('../utility/directories');
-const hasPermission = require('../utility/hasPermission');
+const { filesdir, archivedir, resolveArchiveFolder, uploadsdir, appdir } = require('../utility/directories');
+const getChartExport = require('../utility/getChartExport');
+const { hasPermission } = require('../utility/hasPermission');
 const socket = require('../utility/socket');
 const scheduler = require('./scheduler');
+const getDiagramExport = require('../utility/getDiagramExport');
+const apps = require('./apps');
 
 function serialize(format, data) {
   if (format == 'text') return data;
@@ -18,21 +22,21 @@ function deserialize(format, text) {
 }
 
 module.exports = {
-  list_meta: 'get',
-  async list({ folder }) {
-    if (!hasPermission(`files/${folder}/read`)) return [];
+  list_meta: true,
+  async list({ folder }, req) {
+    if (!hasPermission(`files/${folder}/read`, req)) return [];
     const dir = path.join(filesdir(), folder);
     if (!(await fs.exists(dir))) return [];
     const files = (await fs.readdir(dir)).map(file => ({ folder, file }));
     return files;
   },
 
-  listAll_meta: 'get',
-  async listAll() {
+  listAll_meta: true,
+  async listAll(_params, req) {
     const folders = await fs.readdir(filesdir());
     const res = [];
     for (const folder of folders) {
-      if (!hasPermission(`files/${folder}/read`)) continue;
+      if (!hasPermission(`files/${folder}/read`, req)) continue;
       const dir = path.join(filesdir(), folder);
       const files = (await fs.readdir(dir)).map(file => ({ folder, file }));
       res.push(...files);
@@ -40,52 +44,107 @@ module.exports = {
     return res;
   },
 
-  delete_meta: 'post',
-  async delete({ folder, file }) {
-    if (!hasPermission(`files/${folder}/write`)) return;
+  delete_meta: true,
+  async delete({ folder, file }, req) {
+    if (!hasPermission(`files/${folder}/write`, req)) return false;
     await fs.unlink(path.join(filesdir(), folder, file));
     socket.emitChanged(`files-changed-${folder}`);
     socket.emitChanged(`all-files-changed`);
+    return true;
   },
 
-  rename_meta: 'post',
-  async rename({ folder, file, newFile }) {
-    if (!hasPermission(`files/${folder}/write`)) return;
+  rename_meta: true,
+  async rename({ folder, file, newFile }, req) {
+    if (!hasPermission(`files/${folder}/write`, req)) return false;
     await fs.rename(path.join(filesdir(), folder, file), path.join(filesdir(), folder, newFile));
     socket.emitChanged(`files-changed-${folder}`);
     socket.emitChanged(`all-files-changed`);
+    return true;
   },
 
-  load_meta: 'post',
-  async load({ folder, file, format }) {
-    if (!hasPermission(`files/${folder}/read`)) return null;
-    const text = await fs.readFile(path.join(filesdir(), folder, file), { encoding: 'utf-8' });
+  refresh_meta: true,
+  async refresh({ folders }, req) {
+    for (const folder of folders) {
+      socket.emitChanged(`files-changed-${folder}`);
+      socket.emitChanged(`all-files-changed`);
+    }
+    return true;
+  },
+
+  copy_meta: true,
+  async copy({ folder, file, newFile }, req) {
+    if (!hasPermission(`files/${folder}/write`, req)) return false;
+    await fs.copyFile(path.join(filesdir(), folder, file), path.join(filesdir(), folder, newFile));
+    socket.emitChanged(`files-changed-${folder}`);
+    socket.emitChanged(`all-files-changed`);
+    return true;
+  },
+
+  load_meta: true,
+  async load({ folder, file, format }, req) {
+    if (folder.startsWith('archive:')) {
+      const text = await fs.readFile(path.join(resolveArchiveFolder(folder.substring('archive:'.length)), file), {
+        encoding: 'utf-8',
+      });
+      return deserialize(format, text);
+    } else if (folder.startsWith('app:')) {
+      const text = await fs.readFile(path.join(appdir(), folder.substring('app:'.length), file), {
+        encoding: 'utf-8',
+      });
+      return deserialize(format, text);
+    } else {
+      if (!hasPermission(`files/${folder}/read`, req)) return null;
+      const text = await fs.readFile(path.join(filesdir(), folder, file), { encoding: 'utf-8' });
+      return deserialize(format, text);
+    }
+  },
+
+  loadFrom_meta: true,
+  async loadFrom({ filePath, format }, req) {
+    const text = await fs.readFile(filePath, { encoding: 'utf-8' });
     return deserialize(format, text);
   },
 
-  save_meta: 'post',
-  async save({ folder, file, data, format }) {
-    if (!hasPermission(`files/${folder}/write`)) return;
-    const dir = path.join(filesdir(), folder);
-    if (!(await fs.exists(dir))) {
-      await fs.mkdir(dir);
-    }
-    await fs.writeFile(path.join(dir, file), serialize(format, data));
-    socket.emitChanged(`files-changed-${folder}`);
-    socket.emitChanged(`all-files-changed`);
-    if (folder == 'shell') {
-      scheduler.reload();
+  save_meta: true,
+  async save({ folder, file, data, format }, req) {
+    if (folder.startsWith('archive:')) {
+      if (!hasPermission(`archive/write`, req)) return false;
+      const dir = resolveArchiveFolder(folder.substring('archive:'.length));
+      await fs.writeFile(path.join(dir, file), serialize(format, data));
+      socket.emitChanged(`archive-files-changed-${folder.substring('archive:'.length)}`);
+      return true;
+    } else if (folder.startsWith('app:')) {
+      if (!hasPermission(`apps/write`, req)) return false;
+      const app = folder.substring('app:'.length);
+      await fs.writeFile(path.join(appdir(), app, file), serialize(format, data));
+      socket.emitChanged(`app-files-changed-${app}`);
+      socket.emitChanged('used-apps-changed');
+      apps.emitChangedDbApp(folder);
+      return true;
+    } else {
+      if (!hasPermission(`files/${folder}/write`, req)) return false;
+      const dir = path.join(filesdir(), folder);
+      if (!(await fs.exists(dir))) {
+        await fs.mkdir(dir);
+      }
+      await fs.writeFile(path.join(dir, file), serialize(format, data));
+      socket.emitChanged(`files-changed-${folder}`);
+      socket.emitChanged(`all-files-changed`);
+      if (folder == 'shell') {
+        scheduler.reload();
+      }
+      return true;
     }
   },
 
-  saveAs_meta: 'post',
+  saveAs_meta: true,
   async saveAs({ filePath, data, format }) {
     await fs.writeFile(filePath, serialize(format, data));
   },
 
-  favorites_meta: 'get',
-  async favorites() {
-    if (!hasPermission(`files/favorites/read`)) return [];
+  favorites_meta: true,
+  async favorites(_params, req) {
+    if (!hasPermission(`files/favorites/read`, req)) return [];
     const dir = path.join(filesdir(), 'favorites');
     if (!(await fs.exists(dir))) return [];
     const files = await fs.readdir(dir);
@@ -101,4 +160,56 @@ module.exports = {
     }
     return res;
   },
+
+  generateUploadsFile_meta: true,
+  async generateUploadsFile({ extension }) {
+    const fileName = `${uuidv1()}.${extension || 'html'}`;
+    return {
+      fileName,
+      filePath: path.join(uploadsdir(), fileName),
+    };
+  },
+
+  exportChart_meta: true,
+  async exportChart({ filePath, title, config, image }) {
+    const fileName = path.parse(filePath).base;
+    const imageFile = fileName.replace('.html', '-preview.png');
+    const html = getChartExport(title, config, imageFile);
+    await fs.writeFile(filePath, html);
+    if (image) {
+      const index = image.indexOf('base64,');
+      if (index > 0) {
+        const data = image.substr(index + 'base64,'.length);
+        const buf = Buffer.from(data, 'base64');
+        await fs.writeFile(filePath.replace('.html', '-preview.png'), buf);
+      }
+    }
+    return true;
+  },
+
+  exportDiagram_meta: true,
+  async exportDiagram({ filePath, html, css, themeType, themeClassName }) {
+    await fs.writeFile(filePath, getDiagramExport(html, css, themeType, themeClassName));
+    return true;
+  },
+
+  getFileRealPath_meta: true,
+  async getFileRealPath({ folder, file }, req) {
+    if (folder.startsWith('archive:')) {
+      if (!hasPermission(`archive/write`, req)) return false;
+      const dir = resolveArchiveFolder(folder.substring('archive:'.length));
+      return path.join(dir, file);
+    } else if (folder.startsWith('app:')) {
+      if (!hasPermission(`apps/write`, req)) return false;
+      const app = folder.substring('app:'.length);
+      return path.join(appdir(), app, file);
+    } else {
+      if (!hasPermission(`files/${folder}/write`, req)) return false;
+      const dir = path.join(filesdir(), folder);
+      if (!(await fs.exists(dir))) {
+        await fs.mkdir(dir);
+      }
+      return path.join(dir, file);
+    }
+  },
 };

packages/api/src/controllers/jsldata.js

@@ -1,6 +1,8 @@
+const { filterName } = require('dbgate-tools');
 const fs = require('fs');
 const lineReader = require('line-reader');
 const _ = require('lodash');
+const { __ } = require('lodash/fp');
 const DatastoreProxy = require('../utility/DatastoreProxy');
 const { saveFreeTableData } = require('../utility/freeTableStorage');
 const getJslFileName = require('../utility/getJslFileName');
@@ -10,7 +12,10 @@ const socket = require('../utility/socket');
 function readFirstLine(file) {
   return new Promise((resolve, reject) => {
     lineReader.open(file, (err, reader) => {
-      if (err) reject(err);
+      if (err) {
+        reject(err);
+        return;
+      }
       if (reader.hasNextLine()) {
         reader.nextLine((err, line) => {
           if (err) reject(err);
@@ -104,21 +109,34 @@ module.exports = {
     return datastore;
   },
 
-  getInfo_meta: 'get',
+  getInfo_meta: true,
   async getInfo({ jslid }) {
     const file = getJslFileName(jslid);
-    const firstLine = await readFirstLine(file);
-    if (firstLine) return JSON.parse(firstLine);
-    return null;
+    try {
+      const firstLine = await readFirstLine(file);
+      if (firstLine) {
+        const parsed = JSON.parse(firstLine);
+        if (parsed.__isStreamHeader) {
+          return parsed;
+        }
+        return {
+          __isStreamHeader: true,
+          __isDynamicStructure: true,
+        };
+      }
+      return null;
+    } catch (err) {
+      return null;
+    }
   },
 
-  getRows_meta: 'post',
+  getRows_meta: true,
   async getRows({ jslid, offset, limit, filters }) {
     const datastore = await this.ensureDatastore(jslid);
     return datastore.getRows(offset, limit, _.isEmpty(filters) ? null : filters);
   },
 
-  getStats_meta: 'get',
+  getStats_meta: true,
   getStats({ jslid }) {
     const file = `${getJslFileName(jslid)}.stats`;
     if (fs.existsSync(file)) {
@@ -131,8 +149,21 @@ module.exports = {
     return {};
   },
 
+  loadFieldValues_meta: true,
+  async loadFieldValues({ jslid, field, search }) {
+    const datastore = await this.ensureDatastore(jslid);
+    const res = new Set();
+    await datastore.enumRows(row => {
+      if (!filterName(search, row[field])) return true;
+      res.add(row[field]);
+      return res.size < 100;
+    });
+    // @ts-ignore
+    return [...res].map(value => ({ value }));
+  },
+
   async notifyChangedStats(stats) {
-    console.log('SENDING STATS', JSON.stringify(stats));
+    // console.log('SENDING STATS', JSON.stringify(stats));
     const datastore = this.datastores[stats.jslid];
     if (datastore) await datastore.notifyChanged();
     socket.emit(`jsldata-stats-${stats.jslid}`, stats);
@@ -146,9 +177,15 @@ module.exports = {
     // }
   },
 
-  saveFreeTable_meta: 'post',
+  saveFreeTable_meta: true,
   async saveFreeTable({ jslid, data }) {
     saveFreeTableData(getJslFileName(jslid), data);
     return true;
   },
+
+  saveText_meta: true,
+  async saveText({ jslid, text }) {
+    await fs.promises.writeFile(getJslFileName(jslid), text);
+    return true;
+  },
 };

packages/api/src/controllers/metadata.js

@@ -7,14 +7,14 @@ function pickObjectNames(array) {
 }
 
 module.exports = {
-  // tableData_meta: 'get',
+  // tableData_meta: true,
   // async tableData({ conid, database, schemaName, pureName }) {
   //   const opened = await databaseConnections.ensureOpened(conid, database);
   //   const res = await databaseConnections.sendRequest(opened, { msgtype: 'tableData', schemaName, pureName });
   //   return res;
   // },
 
-  listObjects_meta: 'get',
+  listObjects_meta: true,
   async listObjects({ conid, database }) {
     const opened = await databaseConnections.ensureOpened(conid, database);
     const types = ['tables', 'collections', 'views', 'procedures', 'functions', 'triggers'];
@@ -27,7 +27,7 @@ module.exports = {
     );
   },
 
-  tableInfo_meta: 'get',
+  tableInfo_meta: true,
   async tableInfo({ conid, database, schemaName, pureName }) {
     const opened = await databaseConnections.ensureOpened(conid, database);
     const table = opened.structure.tables.find(x => x.pureName == pureName && x.schemaName == schemaName);
@@ -38,7 +38,7 @@ module.exports = {
     };
   },
 
-  sqlObjectInfo_meta: 'get',
+  sqlObjectInfo_meta: true,
   async sqlObjectInfo({ objectTypeField, conid, database, schemaName, pureName }) {
     const opened = await databaseConnections.ensureOpened(conid, database);
     const res = opened.structure[objectTypeField].find(x => x.pureName == pureName && x.schemaName == schemaName);

packages/api/src/controllers/plugins.js

@@ -7,12 +7,12 @@ const socket = require('../utility/socket');
 const compareVersions = require('compare-versions');
 const requirePlugin = require('../shell/requirePlugin');
 const downloadPackage = require('../utility/downloadPackage');
-const hasPermission = require('../utility/hasPermission');
+const { hasPermission } = require('../utility/hasPermission');
 const _ = require('lodash');
 const packagedPluginsContent = require('../packagedPluginsContent');
 
 module.exports = {
-  script_meta: 'get',
+  script_meta: true,
   async script({ packageName }) {
     const packagedContent = packagedPluginsContent();
 
@@ -30,7 +30,7 @@ module.exports = {
     return data;
   },
 
-  search_meta: 'get',
+  search_meta: true,
   async search({ filter }) {
     // DOCS: https://github.com/npm/registry/blob/master/docs/REGISTRY-API.md#get-v1search
     const resp = await axios.default.get(
@@ -40,7 +40,7 @@ module.exports = {
     return (objects || []).map(x => x.package);
   },
 
-  info_meta: 'get',
+  info_meta: true,
   async info({ packageName }) {
     try {
       const infoResp = await axios.default.get(`https://registry.npmjs.org/${packageName}`);
@@ -63,7 +63,7 @@ module.exports = {
     }
   },
 
-  installed_meta: 'get',
+  installed_meta: true,
   async installed() {
     const packagedContent = packagedPluginsContent();
 
@@ -72,6 +72,7 @@ module.exports = {
 
     const res = [];
     for (const packageName of _.union(files1, files2)) {
+      if (packageName == 'dist') continue;
       if (!/^dbgate-plugin-.*$/.test(packageName)) continue;
       try {
         if (packagedContent && packagedContent[packageName]) {
@@ -88,6 +89,12 @@ module.exports = {
               encoding: 'utf-8',
             })
             .then(x => JSON.parse(x));
+          if (!manifest.keywords) {
+            continue;
+          }
+          if (!manifest.keywords.includes('dbgateplugin')) {
+            continue;
+          }
           const readmeFile = path.join(isPackaged ? packagedPluginsDir() : pluginsdir(), packageName, 'README.md');
           // @ts-ignore
           if (await fs.exists(readmeFile)) {
@@ -107,9 +114,9 @@ module.exports = {
   //   await fs.writeFile(path.join(datadir(), 'removed-plugins'), this.removedPlugins.join('\n'));
   // },
 
-  install_meta: 'post',
-  async install({ packageName }) {
-    if (!hasPermission(`plugins/install`)) return;
+  install_meta: true,
+  async install({ packageName }, req) {
+    if (!hasPermission(`plugins/install`, req)) return;
     const dir = path.join(pluginsdir(), packageName);
     // @ts-ignore
     if (!(await fs.exists(dir))) {
@@ -118,21 +125,23 @@ module.exports = {
     socket.emitChanged(`installed-plugins-changed`);
     // this.removedPlugins = this.removedPlugins.filter(x => x != packageName);
     // await this.saveRemovePlugins();
+    return true;
   },
 
-  uninstall_meta: 'post',
-  async uninstall({ packageName }) {
-    if (!hasPermission(`plugins/install`)) return;
+  uninstall_meta: true,
+  async uninstall({ packageName }, req) {
+    if (!hasPermission(`plugins/install`, req)) return;
     const dir = path.join(pluginsdir(), packageName);
     await fs.rmdir(dir, { recursive: true });
     socket.emitChanged(`installed-plugins-changed`);
     // this.removedPlugins.push(packageName);
-    await this.saveRemovePlugins();
+    // await this.saveRemovePlugins();
+    return true;
   },
 
-  upgrade_meta: 'post',
-  async upgrade({ packageName }) {
-    if (!hasPermission(`plugins/install`)) return;
+  upgrade_meta: true,
+  async upgrade({ packageName }, req) {
+    if (!hasPermission(`plugins/install`, req)) return;
     const dir = path.join(pluginsdir(), packageName);
     // @ts-ignore
     if (await fs.exists(dir)) {
@@ -141,17 +150,19 @@ module.exports = {
     }
 
     socket.emitChanged(`installed-plugins-changed`);
+    return true;
   },
 
-  command_meta: 'post',
+  command_meta: true,
   async command({ packageName, command, args }) {
     const content = requirePlugin(packageName);
     return content.commands[command](args);
   },
 
-  authTypes_meta: 'get',
+  authTypes_meta: true,
   async authTypes({ engine }) {
     const packageName = extractPackageName(engine);
+    if (!packageName) return null;
     const content = requirePlugin(packageName);
     const driver = content.drivers.find(x => x.engine == engine);
     if (!driver || !driver.getAuthTypes) return null;

packages/api/src/controllers/queryHistory.js

@@ -34,7 +34,7 @@ function readCore(reader, skip, limit, filter) {
 }
 
 module.exports = {
-  read_meta: 'get',
+  read_meta: true,
   async read({ skip, limit, filter }) {
     const fileName = path.join(datadir(), 'query-history.jsonl');
     // @ts-ignore
@@ -44,11 +44,11 @@ module.exports = {
     return res;
   },
 
-  write_meta: 'post',
+  write_meta: true,
   async write({ data }) {
     const fileName = path.join(datadir(), 'query-history.jsonl');
     await fs.appendFile(fileName, JSON.stringify(data) + '\n');
-    socket.emitChanged('query-history-changed');
+    socket.emit('query-history-changed');
     return 'OK';
   },
 };

packages/api/src/controllers/runners.js

@@ -6,8 +6,10 @@ const byline = require('byline');
 const socket = require('../utility/socket');
 const { fork } = require('child_process');
 const { rundir, uploadsdir, pluginsdir, getPluginBackendPath, packagedPluginList } = require('../utility/directories');
-const { extractShellApiPlugins, extractShellApiFunctionName } = require('dbgate-tools');
+const { extractShellApiPlugins, extractShellApiFunctionName, jsonScriptToJavascript } = require('dbgate-tools');
 const { handleProcessCommunication } = require('../utility/processComm');
+const processArgs = require('../utility/processArgs');
+const platformInfo = require('../utility/platformInfo');
 
 function extractPlugins(script) {
   const requireRegex = /\s*\/\/\s*@require\s+([^\s]+)\s*\n/g;
@@ -15,17 +17,20 @@ function extractPlugins(script) {
   return matches.map(x => x[1]);
 }
 
-const requirePluginsTemplate = plugins =>
+const requirePluginsTemplate = (plugins, isExport) =>
   plugins
     .map(
-      packageName => `const ${_.camelCase(packageName)} = require(process.env.PLUGIN_${_.camelCase(packageName)});\n`
+      packageName =>
+        `const ${_.camelCase(packageName)} = require(${
+          isExport ? `'${packageName}'` : `process.env.PLUGIN_${_.camelCase(packageName)}`
+        });\n`
     )
     .join('') + `dbgateApi.registerPlugins(${plugins.map(x => _.camelCase(x)).join(',')});\n`;
 
-const scriptTemplate = script => `
-const dbgateApi = require(process.env.DBGATE_API);
+const scriptTemplate = (script, isExport) => `
+const dbgateApi = require(${isExport ? `'dbgate-api'` : 'process.env.DBGATE_API'});
 dbgateApi.initializeApiEnvironment();
-${requirePluginsTemplate(extractPlugins(script))}
+${requirePluginsTemplate(extractPlugins(script), isExport)}
 require=null;
 async function run() {
 ${script}
@@ -95,15 +100,23 @@ module.exports = {
     const pluginNames = _.union(fs.readdirSync(pluginsdir()), packagedPluginList);
     console.log(`RUNNING SCRIPT ${scriptFile}`);
     // const subprocess = fork(scriptFile, ['--checkParent', '--max-old-space-size=8192'], {
-    const subprocess = fork(scriptFile, ['--checkParent', ...process.argv.slice(3)], {
-      cwd: directory,
-      stdio: ['ignore', 'pipe', 'pipe', 'ipc'],
-      env: {
-        ...process.env,
-        DBGATE_API: global['dbgateApiModulePath'] || process.argv[1],
-        ..._.fromPairs(pluginNames.map(name => [`PLUGIN_${_.camelCase(name)}`, getPluginBackendPath(name)])),
-      },
-    });
+    const subprocess = fork(
+      scriptFile,
+      [
+        '--checkParent', // ...process.argv.slice(3)
+        '--is-forked-api',
+        ...processArgs.getPassArgs(),
+      ],
+      {
+        cwd: directory,
+        stdio: ['ignore', 'pipe', 'pipe', 'ipc'],
+        env: {
+          ...process.env,
+          DBGATE_API: global['API_PACKAGE'] || process.argv[1],
+          ..._.fromPairs(pluginNames.map(name => [`PLUGIN_${_.camelCase(name)}`, getPluginBackendPath(name)])),
+        },
+      }
+    );
     const pipeDispatcher = severity => data =>
       this.dispatchMessage(runid, { severity, message: data.toString().trim() });
 
@@ -133,16 +146,31 @@ module.exports = {
       if (handleProcessCommunication(message, subprocess)) return;
       this[`handle_${msgtype}`](runid, message);
     });
-    return newOpened;
+    return _.pick(newOpened, ['runid']);
   },
 
-  start_meta: 'post',
+  start_meta: true,
   async start({ script }) {
     const runid = uuidv1();
-    return this.startCore(runid, scriptTemplate(script));
+
+    if (script.type == 'json') {
+      const js = jsonScriptToJavascript(script);
+      return this.startCore(runid, scriptTemplate(js, false));
+    }
+
+    if (!platformInfo.allowShellScripting) {
+      return { errorMessage: 'Shell scripting is not allowed' };
+    }
+
+    return this.startCore(runid, scriptTemplate(script, false));
   },
 
-  cancel_meta: 'post',
+  getNodeScript_meta: true,
+  async getNodeScript({ script }) {
+    return scriptTemplate(script, true);
+  },
+
+  cancel_meta: true,
   async cancel({ runid }) {
     const runner = this.opened.find(x => x.runid == runid);
     if (!runner) {
@@ -152,7 +180,7 @@ module.exports = {
     return { state: 'ok' };
   },
 
-  files_meta: 'get',
+  files_meta: true,
   async files({ runid }) {
     const directory = path.join(rundir(), runid);
     const files = await fs.readdir(directory);
@@ -168,7 +196,7 @@ module.exports = {
     return res;
   },
 
-  loadReader_meta: 'post',
+  loadReader_meta: true,
   async loadReader({ functionName, props }) {
     const promise = new Promise((resolve, reject) => {
       const runid = uuidv1();

packages/api/src/controllers/scheduler.js

@@ -3,7 +3,7 @@ const fs = require('fs-extra');
 const path = require('path');
 const cron = require('node-cron');
 const runners = require('./runners');
-const hasPermission = require('../utility/hasPermission');
+const { hasPermission } = require('../utility/hasPermission');
 
 const scheduleRegex = /\s*\/\/\s*@schedule\s+([^\n]+)\n/;
 
@@ -26,8 +26,8 @@ module.exports = {
     this.tasks.push(task);
   },
 
-  async reload() {
-    if (!hasPermission('files/shell/read')) return;
+  async reload(_params, req) {
+    if (!hasPermission('files/shell/read', req)) return;
     const shellDir = path.join(filesdir(), 'shell');
     await this.unload();
     if (!(await fs.exists(shellDir))) return;

packages/api/src/controllers/serverConnections.js

@@ -6,6 +6,7 @@ const AsyncLock = require('async-lock');
 const { handleProcessCommunication } = require('../utility/processComm');
 const lock = new AsyncLock();
 const config = require('./config');
+const processArgs = require('../utility/processArgs');
 
 module.exports = {
   opened: [],
@@ -36,11 +37,13 @@ module.exports = {
     const res = await lock.acquire(conid, async () => {
       const existing = this.opened.find(x => x.conid == conid);
       if (existing) return existing;
-      const connection = await connections.get({ conid });
-      const subprocess = fork(process.argv[1], [
+      const connection = await connections.getCore({ conid });
+      const subprocess = fork(global['API_PACKAGE'] || process.argv[1], [
+        '--is-forked-api',
         '--start-process',
         'serverConnectionProcess',
-        ...process.argv.slice(3),
+        ...processArgs.getPassArgs(),
+        // ...process.argv.slice(3),
       ]);
       const newOpened = {
         conid,
@@ -66,7 +69,7 @@ module.exports = {
         if (newOpened.disconnected) return;
         this.close(conid, false);
       });
-      subprocess.send({ msgtype: 'connect', ...connection, globalSettings: config.settingsValue });
+      subprocess.send({ msgtype: 'connect', ...connection, globalSettings: await config.getSettings() });
       return newOpened;
     });
     return res;
@@ -86,25 +89,25 @@ module.exports = {
     }
   },
 
-  disconnect_meta: 'post',
+  disconnect_meta: true,
   async disconnect({ conid }) {
     await this.close(conid, true);
     return { status: 'ok' };
   },
 
-  listDatabases_meta: 'get',
+  listDatabases_meta: true,
   async listDatabases({ conid }) {
     const opened = await this.ensureOpened(conid);
     return opened.databases;
   },
 
-  version_meta: 'get',
+  version_meta: true,
   async version({ conid }) {
     const opened = await this.ensureOpened(conid);
     return opened.version;
   },
 
-  serverStatus_meta: 'get',
+  serverStatus_meta: true,
   async serverStatus() {
     return {
       ...this.closed,
@@ -112,7 +115,7 @@ module.exports = {
     };
   },
 
-  ping_meta: 'post',
+  ping_meta: true,
   async ping({ connections }) {
     await Promise.all(
       _.uniq(connections).map(async conid => {
@@ -128,7 +131,7 @@ module.exports = {
     return { status: 'ok' };
   },
 
-  refresh_meta: 'post',
+  refresh_meta: true,
   async refresh({ conid, keepOpen }) {
     if (!keepOpen) this.close(conid);
 
@@ -136,9 +139,10 @@ module.exports = {
     return { status: 'ok' };
   },
 
-  createDatabase_meta: 'post',
+  createDatabase_meta: true,
   async createDatabase({ conid, name }) {
     const opened = await this.ensureOpened(conid);
+    if (opened.connection.isReadOnly) return false;
     opened.subprocess.send({ msgtype: 'createDatabase', name });
     return { status: 'ok' };
   },

packages/api/src/controllers/sessions.js

@@ -4,7 +4,10 @@ const connections = require('./connections');
 const socket = require('../utility/socket');
 const { fork } = require('child_process');
 const jsldata = require('./jsldata');
+const path = require('path');
 const { handleProcessCommunication } = require('../utility/processComm');
+const processArgs = require('../utility/processArgs');
+const { appdir } = require('../utility/directories');
 
 module.exports = {
   /** @type {import('dbgate-types').OpenedSession[]} */
@@ -45,9 +48,18 @@ module.exports = {
     this.dispatchMessage(sesid, info);
   },
 
-  handle_done(sesid) {
+  handle_done(sesid, props) {
     socket.emit(`session-done-${sesid}`);
-    this.dispatchMessage(sesid, 'Query execution finished');
+    if (!props.skipFinishedMessage) {
+      this.dispatchMessage(sesid, 'Query execution finished');
+    }
+    const session = this.opened.find(x => x.sesid == sesid);
+    if (session.loadingReader_jslid) {
+      socket.emit(`session-jslid-done-${session.loadingReader_jslid}`);
+    }
+    if (session.killOnDone) {
+      this.kill({ sesid });
+    }
   },
 
   handle_recordset(sesid, props) {
@@ -59,13 +71,24 @@ module.exports = {
     jsldata.notifyChangedStats(stats);
   },
 
+  handle_initializeFile(sesid, props) {
+    const { jslid } = props;
+    socket.emit(`session-initialize-file-${jslid}`);
+  },
+
   handle_ping() {},
 
-  create_meta: 'post',
+  create_meta: true,
   async create({ conid, database }) {
     const sesid = uuidv1();
-    const connection = await connections.get({ conid });
-    const subprocess = fork(process.argv[1], ['--start-process', 'sessionProcess', ...process.argv.slice(3)]);
+    const connection = await connections.getCore({ conid });
+    const subprocess = fork(global['API_PACKAGE'] || process.argv[1], [
+      '--is-forked-api',
+      '--start-process',
+      'sessionProcess',
+      ...processArgs.getPassArgs(),
+      // ...process.argv.slice(3),
+    ]);
     const newOpened = {
       conid,
       database,
@@ -81,10 +104,10 @@ module.exports = {
       this[`handle_${msgtype}`](sesid, message);
     });
     subprocess.send({ msgtype: 'connect', ...connection, database });
-    return newOpened;
+    return _.pick(newOpened, ['conid', 'database', 'sesid']);
   },
 
-  executeQuery_meta: 'post',
+  executeQuery_meta: true,
   async executeQuery({ sesid, sql }) {
     const session = this.opened.find(x => x.sesid == sesid);
     if (!session) {
@@ -98,7 +121,30 @@ module.exports = {
     return { state: 'ok' };
   },
 
-  // cancel_meta: 'post',
+  executeReader_meta: true,
+  async executeReader({ conid, database, sql, queryName, appFolder }) {
+    const { sesid } = await this.create({ conid, database });
+    const session = this.opened.find(x => x.sesid == sesid);
+    session.killOnDone = true;
+    const jslid = uuidv1();
+    session.loadingReader_jslid = jslid;
+    const fileName = queryName && appFolder ? path.join(appdir(), appFolder, `${queryName}.query.sql`) : null;
+
+    session.subprocess.send({ msgtype: 'executeReader', sql, fileName, jslid });
+
+    return { jslid };
+  },
+
+  stopLoadingReader_meta: true,
+  async stopLoadingReader({ jslid }) {
+    const session = this.opened.find(x => x.loadingReader_jslid == jslid);
+    if (session) {
+      this.kill({ sesid: session.sesid });
+    }
+    return true;
+  },
+
+  // cancel_meta: true,
   // async cancel({ sesid }) {
   //   const session = this.opened.find((x) => x.sesid == sesid);
   //   if (!session) {
@@ -108,7 +154,7 @@ module.exports = {
   //   return { state: 'ok' };
   // },
 
-  kill_meta: 'post',
+  kill_meta: true,
   async kill({ sesid }) {
     const session = this.opened.find(x => x.sesid == sesid);
     if (!session) {
@@ -119,7 +165,7 @@ module.exports = {
     return { state: 'ok' };
   },
 
-  // runCommand_meta: 'post',
+  // runCommand_meta: true,
   // async runCommand({ conid, database, sql }) {
   //   console.log(`Running SQL command , conid=${conid}, database=${database}, sql=${sql}`);
   //   const opened = await this.ensureOpened(conid, database);

packages/api/src/controllers/uploads.js

@@ -25,4 +25,12 @@ module.exports = {
       });
     });
   },
+
+  get_meta: {
+    method: 'get',
+    raw: true,
+  },
+  get(req, res) {
+    res.sendFile(path.join(uploadsdir(), req.query.file));
+  },
 };

packages/api/src/currentVersion.js

@@ -1,5 +1,5 @@
 
 module.exports = { 
-  version: '4.1.1',
+  version: '5.0.0-alpha.1',
   buildTime: '2021-04-17T07:22:49.702Z'
 };

packages/api/src/index.js

@@ -8,7 +8,7 @@ if (processArgs.startProcess) {
   const proc = require('./proc');
   const module = proc[processArgs.startProcess];
   module.start();
-} else if (!module['parent'] && !processArgs.checkParent) {
+} else if (!processArgs.checkParent && !global['API_PACKAGE']) {
   const main = require('./main');
 
   main.start();

packages/api/src/main.js

@@ -4,12 +4,8 @@ const bodyParser = require('body-parser');
 const fileUpload = require('express-fileupload');
 const http = require('http');
 const cors = require('cors');
-const io = require('socket.io');
-const fs = require('fs');
 const getPort = require('get-port');
-const childProcessChecker = require('./utility/childProcessChecker');
 const path = require('path');
-const crypto = require('crypto');
 
 const useController = require('./utility/useController');
 const socket = require('./utility/socket');
@@ -23,6 +19,7 @@ const runners = require('./controllers/runners');
 const jsldata = require('./controllers/jsldata');
 const config = require('./controllers/config');
 const archive = require('./controllers/archive');
+const apps = require('./controllers/apps');
 const uploads = require('./controllers/uploads');
 const plugins = require('./controllers/plugins');
 const files = require('./controllers/files');
@@ -31,11 +28,9 @@ const queryHistory = require('./controllers/queryHistory');
 
 const { rundir } = require('./utility/directories');
 const platformInfo = require('./utility/platformInfo');
-const processArgs = require('./utility/processArgs');
-const timingSafeCheckToken = require('./utility/timingSafeCheckToken');
-
-let authorization = null;
-let checkLocalhostOrigin = null;
+const getExpressPath = require('./utility/getExpressPath');
+const { getLogins } = require('./utility/hasPermission');
+const _ = require('lodash');
 
 function start() {
   // console.log('process.argv', process.argv);
@@ -43,107 +38,134 @@ function start() {
   const app = express();
 
   const server = http.createServer(app);
-  socket.set(io(server));
 
-  if (process.env.LOGIN && process.env.PASSWORD) {
+  const logins = getLogins();
+  if (logins) {
     app.use(
       basicAuth({
-        users: {
-          [process.env.LOGIN]: process.env.PASSWORD,
-        },
+        users: _.fromPairs(logins.map(x => [x.login, x.password])),
         challenge: true,
         realm: 'DbGate Web App',
       })
     );
   }
 
-  app.use(function (req, res, next) {
-    if (authorization && !timingSafeCheckToken(req.headers.authorization, authorization)) {
-      return res.status(403).json({ error: 'Not authorized!' });
-    }
-    if (checkLocalhostOrigin) {
-      if (
-        req.headers.origin &&
-        req.headers.origin != checkLocalhostOrigin &&
-        req.headers.origin != `http://${checkLocalhostOrigin}`
-      ) {
-        console.log('API origin check FAILED');
-        console.log('HEADERS', { ...req.headers, authorization: '***' });
-        return res.status(403).json({ error: 'Not authorized!' });
-      }
-      if (!req.headers.origin && req.headers.host != checkLocalhostOrigin) {
-        console.log('API host check FAILED');
-        console.log('HEADERS', { ...req.headers, authorization: '***' });
-        return res.status(403).json({ error: 'Not authorized!' });
-      }
-    }
-    next();
+  app.use(cors());
+
+  app.get(getExpressPath('/stream'), async function (req, res) {
+    res.set({
+      'Cache-Control': 'no-cache',
+      'Content-Type': 'text/event-stream',
+      'X-Accel-Buffering': 'no',
+      Connection: 'keep-alive',
+    });
+    res.flushHeaders();
+
+    // Tell the client to retry every 10 seconds if connectivity is lost
+    res.write('retry: 10000\n\n');
+    socket.setSseResponse(res);
   });
 
-  app.use(cors());
   app.use(bodyParser.json({ limit: '50mb' }));
 
   app.use(
-    '/uploads',
+    getExpressPath('/uploads'),
     fileUpload({
       limits: { fileSize: 4 * 1024 * 1024 },
     })
   );
 
-  useController(app, '/connections', connections);
-  useController(app, '/server-connections', serverConnections);
-  useController(app, '/database-connections', databaseConnections);
-  useController(app, '/metadata', metadata);
-  useController(app, '/sessions', sessions);
-  useController(app, '/runners', runners);
-  useController(app, '/jsldata', jsldata);
-  useController(app, '/config', config);
-  useController(app, '/archive', archive);
-  useController(app, '/uploads', uploads);
-  useController(app, '/plugins', plugins);
-  useController(app, '/files', files);
-  useController(app, '/scheduler', scheduler);
-  useController(app, '/query-history', queryHistory);
+  useAllControllers(app, null);
 
   // if (process.env.PAGES_DIRECTORY) {
   //   app.use('/pages', express.static(process.env.PAGES_DIRECTORY));
   // }
 
-  app.use('/runners/data', express.static(rundir()));
+  app.use(getExpressPath('/runners/data'), express.static(rundir()));
 
   if (platformInfo.isDocker) {
     // server static files inside docker container
-    app.use(express.static('/home/dbgate-docker/public'));
-  } else {
-    if (!platformInfo.isNpmDist) {
-      app.get('/', (req, res) => {
-        res.send('DbGate API');
-      });
-    }
-  }
+    app.use(getExpressPath('/'), express.static('/home/dbgate-docker/public'));
 
-  if (processArgs.dynport) {
-    childProcessChecker();
-
-    authorization = crypto.randomBytes(32).toString('hex');
-
-    getPort().then(port => {
-      checkLocalhostOrigin = `localhost:${port}`;
-      server.listen(port, () => {
-        console.log(`DbGate API listening on port ${port}`);
-        process.send({ msgtype: 'listening', port, authorization });
-      });
-    });
+    const port = process.env.PORT || 3000;
+    console.log('DbGate API listening on port (docker build)', port);
+    server.listen(port);
   } else if (platformInfo.isNpmDist) {
-    app.use(express.static(path.join(__dirname, '../../dbgate-web/public')));
-    getPort({ port: 5000 }).then(port => {
+    app.use(getExpressPath('/'), express.static(path.join(__dirname, '../../dbgate-web/public')));
+    getPort({
+      port: parseInt(
+        // @ts-ignore
+        process.env.PORT || 3000
+      ),
+    }).then(port => {
       server.listen(port, () => {
-        console.log(`DbGate API listening on port ${port}`);
+        console.log(`DbGate API listening on port ${port} (NPM build)`);
       });
     });
+  } else if (process.env.DEVWEB) {
+    console.log('__dirname', __dirname);
+    console.log(path.join(__dirname, '../../web/public/build'));
+    app.use(getExpressPath('/'), express.static(path.join(__dirname, '../../web/public')));
+
+    const port = process.env.PORT || 3000;
+    console.log('DbGate API & web listening on port (dev web build)', port);
+    server.listen(port);
   } else {
-    server.listen(3000);
+    app.get(getExpressPath('/'), (req, res) => {
+      res.send('DbGate API');
+    });
+
+    const port = process.env.PORT || 3000;
+    console.log('DbGate API listening on port (dev API build)', port);
+    server.listen(port);
   }
+
+  function shutdown() {
+    console.log('\nShutting down DbGate API server');
+    server.close(() => {
+      console.log('Server shut down, terminating');
+      process.exit(0);
+    });
+    setTimeout(() => {
+      console.log('Server close timeout, terminating');
+      process.exit(0);
+    }, 1000);
+  }
+
+  process.on('SIGINT', shutdown);
+  process.on('SIGTERM', shutdown);
+  process.on('SIGBREAK', shutdown);
 }
 
-module.exports = { start };
+function useAllControllers(app, electron) {
+  useController(app, electron, '/connections', connections);
+  useController(app, electron, '/server-connections', serverConnections);
+  useController(app, electron, '/database-connections', databaseConnections);
+  useController(app, electron, '/metadata', metadata);
+  useController(app, electron, '/sessions', sessions);
+  useController(app, electron, '/runners', runners);
+  useController(app, electron, '/jsldata', jsldata);
+  useController(app, electron, '/config', config);
+  useController(app, electron, '/archive', archive);
+  useController(app, electron, '/uploads', uploads);
+  useController(app, electron, '/plugins', plugins);
+  useController(app, electron, '/files', files);
+  useController(app, electron, '/scheduler', scheduler);
+  useController(app, electron, '/query-history', queryHistory);
+  useController(app, electron, '/apps', apps);
+}
+
+function addElectronSender(electronSender) {
+  socket.addElectronSender(electronSender);
+}
+function removeElectronSender(electronSender) {
+  socket.removeElectronSender(electronSender);
+}
+
+module.exports = {
+  start,
+  useAllControllers,
+  addElectronSender,
+  removeElectronSender,
+  configController: config,
+};

packages/api/src/nativeModules.js

@@ -1,5 +1,13 @@
 const argIndex = process.argv.indexOf('--native-modules');
-const redirectFile = argIndex > 0 ? process.argv[argIndex + 1] : null;
+const redirectFile = global['NATIVE_MODULES'] || (argIndex > 0 ? process.argv[argIndex + 1] : null);
 
-// @ts-ignore
-module.exports = redirectFile ? __non_webpack_require__(redirectFile) : require('./nativeModulesContent');
+function requireDynamic(file) {
+  try {
+    // @ts-ignore
+    return __non_webpack_require__(redirectFile);
+  } catch (err) {
+    return require(redirectFile);
+  }
+}
+
+module.exports = redirectFile ? requireDynamic(redirectFile) : require('./nativeModulesContent');

packages/api/src/proc/connectProcess.js

@@ -2,17 +2,9 @@ const childProcessChecker = require('../utility/childProcessChecker');
 const requireEngineDriver = require('../utility/requireEngineDriver');
 const connectUtility = require('../utility/connectUtility');
 const { handleProcessCommunication } = require('../utility/processComm');
+const { pickSafeConnectionInfo } = require('../utility/crypting');
 const _ = require('lodash');
 
-function pickSafeConnectionInfo(connection) {
-  return _.mapValues(connection, (v, k) => {
-    if (k == 'engine' || k == 'port' || k == 'authType' || k == 'sshMode' || k == 'passwordMode') return v;
-    if (v === null || v === true || v === false) return v;
-    if (v) return '***';
-    return undefined;
-  });
-}
-
 const formatErrorDetail = (e, connection) => `${e.stack}
 
 Error JSON: ${JSON.stringify(e, undefined, 2)}
@@ -28,7 +20,7 @@ function start() {
     if (handleProcessCommunication(connection)) return;
     try {
       const driver = requireEngineDriver(connection);
-      const conn = await connectUtility(driver, connection);
+      const conn = await connectUtility(driver, connection, 'app');
       const res = await driver.getVersion(conn);
       process.send({ msgtype: 'connected', ...res });
     } catch (e) {

packages/api/src/proc/databaseConnectionProcess.js

@@ -6,14 +6,25 @@ const requireEngineDriver = require('../utility/requireEngineDriver');
 const connectUtility = require('../utility/connectUtility');
 const { handleProcessCommunication } = require('../utility/processComm');
 const { SqlGenerator } = require('dbgate-tools');
+const generateDeploySql = require('../shell/generateDeploySql');
+const { dumpSqlSelect } = require('dbgate-sqltree');
 
 let systemConnection;
 let storedConnection;
 let afterConnectCallbacks = [];
+let afterAnalyseCallbacks = [];
 let analysedStructure = null;
 let lastPing = null;
+let lastStatusString = null;
 let lastStatus = null;
 let analysedTime = 0;
+let serverVersion;
+
+let statusCounter = 0;
+function getStatusCounter() {
+  statusCounter += 1;
+  return statusCounter;
+}
 
 async function checkedAsyncCall(promise) {
   try {
@@ -36,19 +47,23 @@ async function handleFullRefresh() {
   loadingModel = true;
   const driver = requireEngineDriver(storedConnection);
   setStatusName('loadStructure');
-  analysedStructure = await checkedAsyncCall(driver.analyseFull(systemConnection));
+  analysedStructure = await checkedAsyncCall(driver.analyseFull(systemConnection, serverVersion));
   analysedTime = new Date().getTime();
   process.send({ msgtype: 'structure', structure: analysedStructure });
   process.send({ msgtype: 'structureTime', analysedTime });
   setStatusName('ok');
+
   loadingModel = false;
+  resolveAnalysedPromises();
 }
 
 async function handleIncrementalRefresh(forceSend) {
   loadingModel = true;
   const driver = requireEngineDriver(storedConnection);
   setStatusName('checkStructure');
-  const newStructure = await checkedAsyncCall(driver.analyseIncremental(systemConnection, analysedStructure));
+  const newStructure = await checkedAsyncCall(
+    driver.analyseIncremental(systemConnection, analysedStructure, serverVersion)
+  );
   analysedTime = new Date().getTime();
   if (newStructure != null) {
     analysedStructure = newStructure;
@@ -61,29 +76,34 @@ async function handleIncrementalRefresh(forceSend) {
   process.send({ msgtype: 'structureTime', analysedTime });
   setStatusName('ok');
   loadingModel = false;
+  resolveAnalysedPromises();
 }
 
-function handleSyncModel() {
+function handleSyncModel({ isFullRefresh }) {
   if (loadingModel) return;
-  handleIncrementalRefresh();
+  if (isFullRefresh) handleFullRefresh();
+  else handleIncrementalRefresh();
 }
 
 function setStatus(status) {
-  const statusString = stableStringify(status);
-  if (lastStatus != statusString) {
-    process.send({ msgtype: 'status', status });
-    lastStatus = statusString;
+  const newStatus = { ...lastStatus, ...status };
+  const statusString = stableStringify(newStatus);
+  if (lastStatusString != statusString) {
+    process.send({ msgtype: 'status', status: { ...newStatus, counter: getStatusCounter() } });
+    lastStatusString = statusString;
+    lastStatus = newStatus;
   }
 }
 
 function setStatusName(name) {
-  setStatus({ name });
+  setStatus({ name, message: null });
 }
 
 async function readVersion() {
   const driver = requireEngineDriver(storedConnection);
   const version = await driver.getVersion(systemConnection);
   process.send({ msgtype: 'version', version });
+  serverVersion = version;
 }
 
 async function handleConnect({ connection, structure, globalSettings }) {
@@ -92,8 +112,9 @@ async function handleConnect({ connection, structure, globalSettings }) {
 
   if (!structure) setStatusName('pending');
   const driver = requireEngineDriver(storedConnection);
-  systemConnection = await checkedAsyncCall(connectUtility(driver, storedConnection));
-  readVersion();
+  systemConnection = await checkedAsyncCall(connectUtility(driver, storedConnection, 'app'));
+  systemConnection.feedback = feedback => setStatus({ feedback });
+  await checkedAsyncCall(readVersion());
   if (structure) {
     analysedStructure = structure;
     handleIncrementalRefresh(true);
@@ -121,10 +142,25 @@ function waitConnected() {
   });
 }
 
+function waitStructure() {
+  if (analysedStructure) return Promise.resolve();
+  return new Promise((resolve, reject) => {
+    afterAnalyseCallbacks.push([resolve, reject]);
+  });
+}
+
+function resolveAnalysedPromises() {
+  for (const [resolve] of afterAnalyseCallbacks) {
+    resolve();
+  }
+  afterAnalyseCallbacks = [];
+}
+
 async function handleRunScript({ msgid, sql }) {
   await waitConnected();
   const driver = requireEngineDriver(storedConnection);
   try {
+    ensureExecuteCustomScript(driver);
     await driver.script(systemConnection, sql);
     process.send({ msgtype: 'response', msgid });
   } catch (err) {
@@ -136,6 +172,7 @@ async function handleQueryData({ msgid, sql }) {
   await waitConnected();
   const driver = requireEngineDriver(storedConnection);
   try {
+    ensureExecuteCustomScript(driver);
     const res = await driver.query(systemConnection, sql);
     process.send({ msgtype: 'response', msgid, ...res });
   } catch (err) {
@@ -143,21 +180,75 @@ async function handleQueryData({ msgid, sql }) {
   }
 }
 
-async function handleCollectionData({ msgid, options }) {
+async function handleSqlSelect({ msgid, select }) {
+  const driver = requireEngineDriver(storedConnection);
+  const dmp = driver.createDumper();
+  dumpSqlSelect(dmp, select);
+  return handleQueryData({ msgid, sql: dmp.s });
+}
+
+async function handleDriverDataCore(msgid, callMethod) {
   await waitConnected();
   const driver = requireEngineDriver(storedConnection);
   try {
-    const result = await driver.readCollection(systemConnection, options);
+    const result = await callMethod(driver);
     process.send({ msgtype: 'response', msgid, result });
   } catch (err) {
     process.send({ msgtype: 'response', msgid, errorMessage: err.message });
   }
 }
 
+async function handleCollectionData({ msgid, options }) {
+  return handleDriverDataCore(msgid, driver => driver.readCollection(systemConnection, options));
+}
+
+async function handleLoadKeys({ msgid, root, filter }) {
+  return handleDriverDataCore(msgid, driver => driver.loadKeys(systemConnection, root, filter));
+}
+
+async function handleExportKeys({ msgid, options }) {
+  return handleDriverDataCore(msgid, driver => driver.exportKeys(systemConnection, options));
+}
+
+async function handleLoadKeyInfo({ msgid, key }) {
+  return handleDriverDataCore(msgid, driver => driver.loadKeyInfo(systemConnection, key));
+}
+
+async function handleCallMethod({ msgid, method, args }) {
+  return handleDriverDataCore(msgid, driver => {
+    if (storedConnection.isReadOnly) {
+      throw new Error('Connection is read only, cannot call custom methods');
+    }
+
+    ensureExecuteCustomScript(driver);
+    return driver.callMethod(systemConnection, method, args);
+  });
+}
+
+async function handleLoadKeyTableRange({ msgid, key, cursor, count }) {
+  return handleDriverDataCore(msgid, driver => driver.loadKeyTableRange(systemConnection, key, cursor, count));
+}
+
+async function handleLoadFieldValues({ msgid, schemaName, pureName, field, search }) {
+  return handleDriverDataCore(msgid, driver =>
+    driver.loadFieldValues(systemConnection, { schemaName, pureName }, field, search)
+  );
+}
+
+function ensureExecuteCustomScript(driver) {
+  if (driver.readOnlySessions) {
+    return;
+  }
+  if (storedConnection.isReadOnly) {
+    throw new Error('Connection is read only');
+  }
+}
+
 async function handleUpdateCollection({ msgid, changeSet }) {
   await waitConnected();
   const driver = requireEngineDriver(storedConnection);
   try {
+    ensureExecuteCustomScript(driver);
     const result = await driver.updateCollection(systemConnection, changeSet);
     process.send({ msgtype: 'response', msgid, result });
   } catch (err) {
@@ -166,7 +257,7 @@ async function handleUpdateCollection({ msgid, changeSet }) {
 }
 
 async function handleSqlPreview({ msgid, objects, options }) {
-  await waitConnected();
+  await waitStructure();
   const driver = requireEngineDriver(storedConnection);
 
   try {
@@ -186,6 +277,22 @@ async function handleSqlPreview({ msgid, objects, options }) {
   }
 }
 
+async function handleGenerateDeploySql({ msgid, modelFolder }) {
+  await waitStructure();
+
+  try {
+    const res = await generateDeploySql({
+      systemConnection,
+      connection: storedConnection,
+      analysedStructure,
+      modelFolder,
+    });
+    process.send({ ...res, msgtype: 'response', msgid });
+  } catch (err) {
+    process.send({ msgtype: 'response', msgid, isError: true, errorMessage: err.message });
+  }
+}
+
 // async function handleRunCommand({ msgid, sql }) {
 //   await waitConnected();
 //   const driver = engines(storedConnection);
@@ -203,9 +310,17 @@ const messageHandlers = {
   runScript: handleRunScript,
   updateCollection: handleUpdateCollection,
   collectionData: handleCollectionData,
+  loadKeys: handleLoadKeys,
+  loadKeyInfo: handleLoadKeyInfo,
+  callMethod: handleCallMethod,
+  loadKeyTableRange: handleLoadKeyTableRange,
   sqlPreview: handleSqlPreview,
   ping: handlePing,
   syncModel: handleSyncModel,
+  generateDeploySql: handleGenerateDeploySql,
+  loadFieldValues: handleLoadFieldValues,
+  sqlSelect: handleSqlSelect,
+  exportKeys: handleExportKeys,
   // runCommand: handleRunCommand,
 };
 
@@ -230,6 +345,7 @@ function start() {
     try {
       await handleMessage(message);
     } catch (e) {
+      console.error('Error in DB connection', e);
       process.send({ msgtype: 'error', error: e.message });
     }
   });

packages/api/src/proc/index.js

@@ -3,6 +3,7 @@ const databaseConnectionProcess = require('./databaseConnectionProcess');
 const serverConnectionProcess = require('./serverConnectionProcess');
 const sessionProcess = require('./sessionProcess');
 const jslDatastoreProcess = require('./jslDatastoreProcess');
+const sshForwardProcess = require('./sshForwardProcess');
 
 module.exports = {
   connectProcess,
@@ -10,4 +11,5 @@ module.exports = {
   serverConnectionProcess,
   sessionProcess,
   jslDatastoreProcess,
+  sshForwardProcess,
 };

packages/api/src/proc/serverConnectionProcess.js

@@ -58,11 +58,14 @@ async function handleConnect(connection) {
 
   const driver = requireEngineDriver(storedConnection);
   try {
-    systemConnection = await connectUtility(driver, storedConnection);
+    systemConnection = await connectUtility(driver, storedConnection, 'app');
     readVersion();
     handleRefresh();
     if (extractBoolSettingsValue(globalSettings, 'connection.autoRefresh', false)) {
-      setInterval(handleRefresh, extractIntSettingsValue(globalSettings, 'connection.autoRefreshInterval', 30, 5, 3600) * 1000);
+      setInterval(
+        handleRefresh,
+        extractIntSettingsValue(globalSettings, 'connection.autoRefreshInterval', 30, 5, 3600) * 1000
+      );
     }
   } catch (err) {
     setStatus({
@@ -80,7 +83,7 @@ function handlePing() {
 
 async function handleCreateDatabase({ name }) {
   const driver = requireEngineDriver(storedConnection);
-  systemConnection = await connectUtility(driver, storedConnection);
+  systemConnection = await connectUtility(driver, storedConnection, 'app');
   console.log(`RUNNING SCRIPT: CREATE DATABASE ${driver.dialect.quoteIdentifier(name)}`);
   if (driver.createDatabase) {
     await driver.createDatabase(systemConnection, name);

packages/api/src/proc/sessionProcess.js

@@ -17,11 +17,15 @@ let afterConnectCallbacks = [];
 // let currentHandlers = [];
 
 class TableWriter {
-  constructor(structure, resultIndex) {
-    this.jslid = uuidv1();
-    this.currentFile = path.join(jsldir(), `${this.jslid}.jsonl`);
+  constructor() {
     this.currentRowCount = 0;
     this.currentChangeIndex = 1;
+    this.initializedFile = false;
+  }
+
+  initializeFromQuery(structure, resultIndex) {
+    this.jslid = uuidv1();
+    this.currentFile = path.join(jsldir(), `${this.jslid}.jsonl`);
     fs.writeFileSync(
       this.currentFile,
       JSON.stringify({
@@ -32,13 +36,21 @@ class TableWriter {
     this.currentStream = fs.createWriteStream(this.currentFile, { flags: 'a' });
     this.writeCurrentStats(false, false);
     this.resultIndex = resultIndex;
+    this.initializedFile = true;
     process.send({ msgtype: 'recordset', jslid: this.jslid, resultIndex });
   }
 
+  initializeFromReader(jslid) {
+    this.jslid = jslid;
+    this.currentFile = path.join(jsldir(), `${this.jslid}.jsonl`);
+    this.writeCurrentStats(false, false);
+  }
+
   row(row) {
     // console.log('ACCEPT ROW', row);
     this.currentStream.write(JSON.stringify(row) + '\n');
     this.currentRowCount += 1;
+
     if (!this.plannedStats) {
       this.plannedStats = true;
       process.nextTick(() => {
@@ -49,6 +61,21 @@ class TableWriter {
     }
   }
 
+  rowFromReader(row) {
+    if (!this.initializedFile) {
+      process.send({ msgtype: 'initializeFile', jslid: this.jslid });
+      this.initializedFile = true;
+
+      fs.writeFileSync(this.currentFile, JSON.stringify(row) + '\n');
+      this.currentStream = fs.createWriteStream(this.currentFile, { flags: 'a' });
+      this.writeCurrentStats(false, false);
+      this.initializedFile = true;
+      return;
+    }
+
+    this.row(row);
+  }
+
   writeCurrentStats(isFinished = false, emitEvent = false) {
     const stats = {
       rowCount: this.currentRowCount,
@@ -63,10 +90,11 @@ class TableWriter {
     }
   }
 
-  close() {
+  close(afterClose) {
     if (this.currentStream) {
       this.currentStream.end(() => {
         this.writeCurrentStats(true, true);
+        if (afterClose) afterClose();
       });
     }
   }
@@ -98,7 +126,11 @@ class StreamHandler {
 
   recordset(columns) {
     this.closeCurrentWriter();
-    this.currentWriter = new TableWriter(Array.isArray(columns) ? { columns } : columns, this.resultIndexHolder.value);
+    this.currentWriter = new TableWriter();
+    this.currentWriter.initializeFromQuery(
+      Array.isArray(columns) ? { columns } : columns,
+      this.resultIndexHolder.value
+    );
     this.resultIndexHolder.value += 1;
 
     // this.writeCurrentStats();
@@ -110,7 +142,6 @@ class StreamHandler {
     // }, 500);
   }
   row(row) {
-    // console.log('ACCEPT ROW', row);
     if (this.currentWriter) this.currentWriter.row(row);
     else if (row.message) process.send({ msgtype: 'info', info: { message: row.message } });
     // this.onRow(this.jslid);
@@ -135,11 +166,22 @@ function handleStream(driver, resultIndexHolder, sql) {
   });
 }
 
+function allowExecuteCustomScript(driver) {
+  if (driver.readOnlySessions) {
+    return true;
+  }
+  if (storedConnection.isReadOnly) {
+    return false;
+    // throw new Error('Connection is read only');
+  }
+  return true;
+}
+
 async function handleConnect(connection) {
   storedConnection = connection;
 
   const driver = requireEngineDriver(storedConnection);
-  systemConnection = await connectUtility(driver, storedConnection);
+  systemConnection = await connectUtility(driver, storedConnection, 'app');
   for (const [resolve] of afterConnectCallbacks) {
     resolve();
   }
@@ -163,6 +205,19 @@ async function handleExecuteQuery({ sql }) {
   await waitConnected();
   const driver = requireEngineDriver(storedConnection);
 
+  if (!allowExecuteCustomScript(driver)) {
+    process.send({
+      msgtype: 'info',
+      info: {
+        message: 'Connection without read-only sessions is read only',
+        severity: 'error',
+      },
+    });
+    process.send({ msgtype: 'done', skipFinishedMessage: true });
+    return;
+    //process.send({ msgtype: 'error', error: e.message });
+  }
+
   const resultIndexHolder = {
     value: 0,
   };
@@ -176,9 +231,39 @@ async function handleExecuteQuery({ sql }) {
   process.send({ msgtype: 'done' });
 }
 
+async function handleExecuteReader({ jslid, sql, fileName }) {
+  await waitConnected();
+
+  const driver = requireEngineDriver(storedConnection);
+
+  if (fileName) {
+    sql = fs.readFileSync(fileName, 'utf-8');
+  } else {
+    if (!allowExecuteCustomScript(driver)) {
+      process.send({ msgtype: 'done' });
+      return;
+    }
+  }
+
+  const writer = new TableWriter();
+  writer.initializeFromReader(jslid);
+
+  const reader = await driver.readQuery(systemConnection, sql);
+
+  reader.on('data', data => {
+    writer.rowFromReader(data);
+  });
+  reader.on('end', () => {
+    writer.close(() => {
+      process.send({ msgtype: 'done' });
+    });
+  });
+}
+
 const messageHandlers = {
   connect: handleConnect,
   executeQuery: handleExecuteQuery,
+  executeReader: handleExecuteReader,
   // cancel: handleCancel,
 };
 

packages/api/src/proc/sshForwardProcess.js

@@ -0,0 +1,68 @@
+const fs = require('fs-extra');
+const platformInfo = require('../utility/platformInfo');
+const childProcessChecker = require('../utility/childProcessChecker');
+const { SSHConnection } = require('node-ssh-forward');
+const { handleProcessCommunication } = require('../utility/processComm');
+
+async function getSshConnection(connection) {
+  const sshConfig = {
+    endHost: connection.sshHost || '',
+    endPort: connection.sshPort || 22,
+    bastionHost: connection.sshBastionHost || '',
+    agentForward: connection.sshMode == 'agent',
+    passphrase: connection.sshMode == 'keyFile' ? connection.sshKeyfilePassword : undefined,
+    username: connection.sshLogin,
+    password: connection.sshMode == 'userPassword' ? connection.sshPassword : undefined,
+    agentSocket: connection.sshMode == 'agent' ? platformInfo.sshAuthSock : undefined,
+    privateKey:
+      connection.sshMode == 'keyFile' && connection.sshKeyfile ? await fs.readFile(connection.sshKeyfile) : undefined,
+    skipAutoPrivateKey: true,
+    noReadline: true,
+  };
+
+  const sshConn = new SSHConnection(sshConfig);
+  return sshConn;
+}
+
+async function handleStart({ connection, tunnelConfig }) {
+  try {
+    const sshConn = await getSshConnection(connection);
+    await sshConn.forward(tunnelConfig);
+
+    process.send({
+      msgtype: 'connected',
+      connection,
+      tunnelConfig,
+    });
+  } catch (err) {
+    process.send({
+      msgtype: 'error',
+      connection,
+      tunnelConfig,
+      errorMessage: err.message,
+    });
+  }
+}
+
+const messageHandlers = {
+  connect: handleStart,
+};
+
+async function handleMessage({ msgtype, ...other }) {
+  const handler = messageHandlers[msgtype];
+  await handler(other);
+}
+
+function start() {
+  childProcessChecker();
+  process.on('message', async message => {
+    if (handleProcessCommunication(message)) return;
+    try {
+      await handleMessage(message);
+    } catch (e) {
+      console.error('sshForwardProcess - unhandled error', e);
+    }
+  });
+}
+
+module.exports = { start };

packages/api/src/shell/archiveReader.js

@@ -1,9 +1,9 @@
 const path = require('path');
-const { archivedir } = require('../utility/directories');
+const { archivedir, resolveArchiveFolder } = require('../utility/directories');
 const jsonLinesReader = require('./jsonLinesReader');
 
 function archiveReader({ folderName, fileName, ...other }) {
-  const jsonlFile = path.join(archivedir(), folderName, `${fileName}.jsonl`);
+  const jsonlFile = path.join(resolveArchiveFolder(folderName), `${fileName}.jsonl`);
   const res = jsonLinesReader({ fileName: jsonlFile, ...other });
   return res;
 }

packages/api/src/shell/archiveWriter.js

@@ -1,11 +1,11 @@
 const path = require('path');
 const fs = require('fs');
-const { archivedir } = require('../utility/directories');
+const { archivedir, resolveArchiveFolder } = require('../utility/directories');
 // const socket = require('../utility/socket');
 const jsonLinesWriter = require('./jsonLinesWriter');
 
 function archiveWriter({ folderName, fileName }) {
-  const dir = path.join(archivedir(), folderName);
+  const dir = resolveArchiveFolder(folderName);
   if (!fs.existsSync(dir)) {
     console.log(`Creating directory ${dir}`);
     fs.mkdirSync(dir);

packages/api/src/shell/copyStream.js

@@ -1,13 +1,47 @@
 const EnsureStreamHeaderStream = require('../utility/EnsureStreamHeaderStream');
+const Stream = require('stream');
+const ColumnMapTransformStream = require('../utility/ColumnMapTransformStream');
+
+function copyStream(input, output, options) {
+  const { columns } = options || {};
+
+  const transforms = [];
+  if (columns) {
+    transforms.push(new ColumnMapTransformStream(columns));
+  }
+  if (output.requireFixedStructure) {
+    transforms.push(new EnsureStreamHeaderStream());
+  }
+
+  // return new Promise((resolve, reject) => {
+  //   Stream.pipeline(input, ...transforms, output, err => {
+  //     if (err) {
+  //       reject(err);
+  //     } else {
+  //       resolve();
+  //     }
+  //   });
+  // });
 
-function copyStream(input, output) {
   return new Promise((resolve, reject) => {
-    const ensureHeader = new EnsureStreamHeaderStream();
     const finisher = output['finisher'] || output;
     finisher.on('finish', resolve);
     finisher.on('error', reject);
-    input.pipe(ensureHeader);
-    ensureHeader.pipe(output);
+
+    let lastStream = input;
+    for (const tran of transforms) {
+      lastStream.pipe(tran);
+      lastStream = tran;
+    }
+    lastStream.pipe(output);
+
+    //  if (output.requireFixedStructure) {
+    //   const ensureHeader = new EnsureStreamHeaderStream();
+    //   input.pipe(ensureHeader);
+    //   ensureHeader.pipe(output);
+    // } else {
+    //   input.pipe(output);
+    // }
   });
 }
 

packages/api/src/shell/deployDb.js

@@ -0,0 +1,17 @@
+const generateDeploySql = require('./generateDeploySql');
+const executeQuery = require('./executeQuery');
+
+async function deployDb({ connection, systemConnection, driver, analysedStructure, modelFolder, loadedDbModel }) {
+  const { sql } = await generateDeploySql({
+    connection,
+    systemConnection,
+    driver,
+    analysedStructure,
+    modelFolder,
+    loadedDbModel,
+  });
+  // console.log('RUNNING DEPLOY SCRIPT:', sql);
+  await executeQuery({ connection, systemConnection, driver, sql });
+}
+
+module.exports = deployDb;

packages/api/src/shell/dumpDatabase.js

@@ -0,0 +1,38 @@
+const requireEngineDriver = require('../utility/requireEngineDriver');
+const connectUtility = require('../utility/connectUtility');
+
+function doDump(dumper) {
+  return new Promise((resolve, reject) => {
+    dumper.once('end', () => {
+      resolve(true);
+    });
+    dumper.once('error', err => {
+      reject(err);
+    });
+    dumper.run();
+  });
+}
+
+async function dumpDatabase({
+  connection = undefined,
+  systemConnection = undefined,
+  driver = undefined,
+  outputFile,
+  databaseName,
+  schemaName,
+}) {
+  console.log(`Dumping database`);
+
+  if (!driver) driver = requireEngineDriver(connection);
+  const pool = systemConnection || (await connectUtility(driver, connection, 'read', { forceRowsAsObjects: true }));
+  console.log(`Connected.`);
+
+  const dumper = await driver.createBackupDumper(pool, {
+    outputFile,
+    databaseName,
+    schemaName,
+  });
+  await doDump(dumper);
+}
+
+module.exports = dumpDatabase;

packages/api/src/shell/executeQuery.js

@@ -1,20 +1,14 @@
-const { splitQuery } = require('dbgate-query-splitter');
 const requireEngineDriver = require('../utility/requireEngineDriver');
 const connectUtility = require('../utility/connectUtility');
 
-async function executeQuery({ connection, sql }) {
+async function executeQuery({ connection = undefined, systemConnection = undefined, driver = undefined, sql }) {
   console.log(`Execute query ${sql}`);
 
-  const driver = requireEngineDriver(connection);
-  const pool = await connectUtility(driver, connection);
+  if (!driver) driver = requireEngineDriver(connection);
+  const pool = systemConnection || (await connectUtility(driver, connection, 'script'));
   console.log(`Connected.`);
 
-  for (const sqlItem of splitQuery(sql, driver.getQuerySplitterOptions('script'))) {
-    console.log('Executing query', sqlItem);
-    await driver.query(pool, sqlItem, { discardResult: true });
-  }
-
-  console.log(`Query finished`);
+  await driver.script(pool, sql);
 }
 
 module.exports = executeQuery;

packages/api/src/shell/fakeObjectReader.js

@@ -5,7 +5,7 @@ async function fakeObjectReader({ delay = 0 } = {}) {
     objectMode: true,
   });
   function doWrite() {
-    pass.write({ columns: [{ columnName: 'id' }, { columnName: 'country' }] });
+    pass.write({ columns: [{ columnName: 'id' }, { columnName: 'country' }], __isStreamHeader: true });
     pass.write({ id: 1, country: 'Czechia' });
     pass.write({ id: 2, country: 'Austria' });
     pass.write({ country: 'Germany', id: 3 });

packages/api/src/shell/generateDeploySql.js

@@ -0,0 +1,61 @@
+const {
+  getAlterDatabaseScript,
+  generateDbPairingId,
+  matchPairedObjects,
+  databaseInfoFromYamlModel,
+  extendDatabaseInfo,
+  modelCompareDbDiffOptions,
+  enrichWithPreloadedRows,
+} = require('dbgate-tools');
+const importDbModel = require('../utility/importDbModel');
+const requireEngineDriver = require('../utility/requireEngineDriver');
+const connectUtility = require('../utility/connectUtility');
+
+async function generateDeploySql({
+  connection,
+  systemConnection = undefined,
+  driver = undefined,
+  analysedStructure = undefined,
+  modelFolder = undefined,
+  loadedDbModel = undefined,
+}) {
+  if (!driver) driver = requireEngineDriver(connection);
+
+  const pool = systemConnection || (await connectUtility(driver, connection, 'read'));
+  if (!analysedStructure) {
+    analysedStructure = await driver.analyseFull(pool);
+  }
+
+  const deployedModel = generateDbPairingId(
+    extendDatabaseInfo(loadedDbModel ? databaseInfoFromYamlModel(loadedDbModel) : await importDbModel(modelFolder))
+  );
+  const currentModel = generateDbPairingId(extendDatabaseInfo(analysedStructure));
+  const opts = {
+    ...modelCompareDbDiffOptions,
+
+    noDropTable: true,
+    noDropColumn: true,
+    noDropConstraint: true,
+    noDropSqlObject: true,
+    noRenameTable: true,
+    noRenameColumn: true,
+  };
+  const currentModelPaired = matchPairedObjects(deployedModel, currentModel, opts);
+  const currentModelPairedPreloaded = await enrichWithPreloadedRows(deployedModel, currentModelPaired, pool, driver);
+
+  // console.log('currentModelPairedPreloaded', currentModelPairedPreloaded.tables[0]);
+  // console.log('deployedModel', deployedModel.tables[0]);
+  // console.log('currentModel', currentModel.tables[0]);
+  // console.log('currentModelPaired', currentModelPaired.tables[0]);
+  const res = getAlterDatabaseScript(
+    currentModelPairedPreloaded,
+    deployedModel,
+    opts,
+    currentModelPairedPreloaded,
+    deployedModel,
+    driver
+  );
+  return res;
+}
+
+module.exports = generateDeploySql;

packages/api/src/shell/importDatabase.js

@@ -0,0 +1,57 @@
+const fs = require('fs');
+const requireEngineDriver = require('../utility/requireEngineDriver');
+const connectUtility = require('../utility/connectUtility');
+const { splitQueryStream } = require('dbgate-query-splitter/lib/splitQueryStream');
+const download = require('./download');
+const stream = require('stream');
+
+class ImportStream extends stream.Transform {
+  constructor(pool, driver) {
+    super({ objectMode: true });
+    this.pool = pool;
+    this.driver = driver;
+  }
+  async _transform(chunk, encoding, cb) {
+    try {
+      await this.driver.script(this.pool, chunk);
+    } catch (err) {
+      this.emit('error', err.message);
+    }
+    cb();
+  }
+  _flush(cb) {
+    this.push('finish');
+    cb();
+    this.emit('end');
+  }
+}
+
+function awaitStreamEnd(stream) {
+  return new Promise((resolve, reject) => {
+    stream.once('end', () => {
+      resolve(true);
+    });
+    stream.once('error', err => {
+      reject(err);
+    });
+  });
+}
+
+async function importDatabase({ connection = undefined, systemConnection = undefined, driver = undefined, inputFile }) {
+  console.log(`Importing database`);
+
+  if (!driver) driver = requireEngineDriver(connection);
+  const pool = systemConnection || (await connectUtility(driver, connection, 'write'));
+  console.log(`Connected.`);
+
+  const downloadedFile = await download(inputFile);
+
+  const fileStream = fs.createReadStream(downloadedFile, 'utf-8');
+  const splittedStream = splitQueryStream(fileStream, driver.getQuerySplitterOptions());
+  const importStream = new ImportStream(pool, driver);
+  // @ts-ignore
+  splittedStream.pipe(importStream);
+  await awaitStreamEnd(importStream);
+}
+
+module.exports = importDatabase;

packages/api/src/shell/index.js

@@ -19,7 +19,10 @@ const requirePlugin = require('./requirePlugin');
 const download = require('./download');
 const executeQuery = require('./executeQuery');
 const loadFile = require('./loadFile');
+const deployDb = require('./deployDb');
 const initializeApiEnvironment = require('./initializeApiEnvironment');
+const dumpDatabase = require('./dumpDatabase');
+const importDatabase = require('./importDatabase');
 
 const dbgateApi = {
   queryReader,
@@ -42,7 +45,10 @@ const dbgateApi = {
   registerPlugins,
   executeQuery,
   loadFile,
+  deployDb,
   initializeApiEnvironment,
+  dumpDatabase,
+  importDatabase,
 };
 
 requirePlugin.initializeDbgateApi(dbgateApi);

packages/api/src/shell/jsonArrayWriter.js

@@ -11,10 +11,7 @@ class StringifyStream extends stream.Transform {
     let skip = false;
 
     if (!this.wasHeader) {
-      skip =
-        chunk.__isStreamHeader ||
-        // TODO remove isArray test
-        Array.isArray(chunk.columns);
+      skip = chunk.__isStreamHeader;
       this.wasHeader = true;
     }
     if (!skip) {

packages/api/src/shell/jsonLinesReader.js

@@ -12,14 +12,14 @@ class ParseStream extends stream.Transform {
   _transform(chunk, encoding, done) {
     const obj = JSON.parse(chunk);
     if (!this.wasHeader) {
-      if (
-        !obj.__isStreamHeader &&
-        // TODO remove isArray test
-        !Array.isArray(obj.columns)
-      ) {
-        this.push({ columns: Object.keys(obj).map(columnName => ({ columnName })) });
+      if (!obj.__isStreamHeader) {
+        this.push({
+          __isStreamHeader: true,
+          __isDynamicStructure: true,
+          // columns: Object.keys(obj).map(columnName => ({ columnName })),
+        });
       }
-      
+
       this.wasHeader = true;
     }
     if (!this.limitRows || this.rowsWritten < this.limitRows) {

packages/api/src/shell/jsonLinesWriter.js

@@ -10,11 +10,7 @@ class StringifyStream extends stream.Transform {
   _transform(chunk, encoding, done) {
     let skip = false;
     if (!this.wasHeader) {
-      skip =
-        (chunk.__isStreamHeader ||
-          // TODO remove isArray test
-          Array.isArray(chunk.columns)) &&
-        !this.header;
+      skip = (chunk.__isStreamHeader && !this.header) || (chunk.__isStreamHeader && chunk.__isDynamicStructure);
       this.wasHeader = true;
     }
     if (!skip) {

packages/api/src/shell/queryReader.js

@@ -1,14 +1,26 @@
 const requireEngineDriver = require('../utility/requireEngineDriver');
-const { decryptConnection } = require('../utility/crypting');
 const connectUtility = require('../utility/connectUtility');
 
-async function queryReader({ connection, sql }) {
-  console.log(`Reading query ${sql}`);
+async function queryReader({
+  connection,
+  query,
+  queryType,
+  // obsolete; use query instead
+  sql,
+}) {
+  // if (sql && json) {
+  //   throw new Error('Only one of sql or json could be set');
+  // }
+  // if (!sql && !json) {
+  //   throw new Error('One of sql or json must be set');
+  // }
+  console.log(`Reading query ${query || sql}`);
+  // else console.log(`Reading query ${JSON.stringify(json)}`);
 
   const driver = requireEngineDriver(connection);
-  const pool = await connectUtility(driver, connection);
+  const pool = await connectUtility(driver, connection, queryType == 'json' ? 'read' : 'script');
   console.log(`Connected.`);
-  return await driver.readQuery(pool, sql);
+  return queryType == 'json' ? await driver.readJsonQuery(pool, query) : await driver.readQuery(pool, query || sql);
 }
 
 module.exports = queryReader;

packages/api/src/shell/sqlDataWriter.js

@@ -5,20 +5,17 @@ const { driverBase } = require('dbgate-tools');
 const requireEngineDriver = require('../utility/requireEngineDriver');
 
 class SqlizeStream extends stream.Transform {
-  constructor({ fileName }) {
+  constructor({ fileName, dataName }) {
     super({ objectMode: true });
     this.wasHeader = false;
     this.tableName = path.parse(fileName).name;
+    this.dataName = dataName;
     this.driver = driverBase;
   }
   _transform(chunk, encoding, done) {
     let skip = false;
     if (!this.wasHeader) {
-      if (
-        chunk.__isStreamHeader ||
-        // TODO remove isArray test
-        Array.isArray(chunk.columns)
-      ) {
+      if (chunk.__isStreamHeader) {
         skip = true;
         this.tableName = chunk.pureName;
         if (chunk.engine) {
@@ -32,7 +29,7 @@ class SqlizeStream extends stream.Transform {
       const dmp = this.driver.createDumper();
       dmp.put(
         '^insert ^into %f (%,i) ^values (%,v);\n',
-        { pureName: this.tableName },
+        { pureName: this.dataName || this.tableName },
         Object.keys(chunk),
         Object.values(chunk)
       );
@@ -42,9 +39,9 @@ class SqlizeStream extends stream.Transform {
   }
 }
 
-async function sqlDataWriter({ fileName, driver, encoding = 'utf-8' }) {
+async function sqlDataWriter({ fileName, dataName, driver, encoding = 'utf-8' }) {
   console.log(`Writing file ${fileName}`);
-  const stringify = new SqlizeStream({ fileName });
+  const stringify = new SqlizeStream({ fileName, dataName });
   const fileStream = fs.createWriteStream(fileName, encoding);
   stringify.pipe(fileStream);
   stringify['finisher'] = fileStream;

packages/api/src/shell/tableReader.js

@@ -4,12 +4,12 @@ const connectUtility = require('../utility/connectUtility');
 
 async function tableReader({ connection, pureName, schemaName }) {
   const driver = requireEngineDriver(connection);
-  const pool = await connectUtility(driver, connection);
+  const pool = await connectUtility(driver, connection, 'read');
   console.log(`Connected.`);
 
   const fullName = { pureName, schemaName };
 
-  if (driver.dialect.nosql) {
+  if (driver.databaseEngineTypes.includes('document')) {
     // @ts-ignore
     console.log(`Reading collection ${fullNameToString(fullName)}`);
     // @ts-ignore