v4.7.4

This reverts commit 074a75075d.

.github/FUNDING.yml

@@ -0,0 +1,12 @@
+# These are supported funding model platforms
+
+github: dbgate
+patreon: # Replace with a single Patreon username
+open_collective: dbgate
+ko_fi: # Replace with a single Ko-fi username
+tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
+community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
+liberapay: # Replace with a single Liberapay username
+issuehunt: # Replace with a single IssueHunt username
+otechie: # Replace with a single Otechie username
+custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2']

.github/ISSUE_TEMPLATE/bug_report.md

@@ -29,5 +29,8 @@ If applicable, add screenshots to help explain your problem.
  - Type - Web/Application
  - Database engine: [e.g. MySQL/PostgreSQL/SQL Server]
 
-**Additional context**
-Anything else you think might be helpful
+**Additional info**
+You could provide additional context, to better understand your case.
+ - How looks your typical DbGate usage?
+ - What other DB software do you use?
+ - Anything else you think might be helpful

.github/ISSUE_TEMPLATE/feature_request.md

@@ -16,5 +16,8 @@ A clear and concise description of what you want to happen.
 **Describe alternatives you've considered**
 A clear and concise description of any alternative solutions or features you've considered.
 
-**Additional context**
-Add any other context or screenshots about the feature request here.
+**Additional info**
+You could provide additional context, to better understand your case.
+ - How looks your typical DbGate usage?
+ - What other DB software do you use?
+ - Anything else you think might be helpful

.github/ISSUE_TEMPLATE/question.md

@@ -16,3 +16,9 @@ App Version [help -> about]:
 
 **Screenshot [if appropriate]**:
 A screenshot of the app if that helps
+
+**Additional info**
+You could provide additional context, to better understand your case.
+ - How looks your typical DbGate usage?
+ - What other DB software do you use?
+ - Anything else you think might be helpful

.github/workflows/build-app-beta.yaml

@@ -1,4 +1,4 @@
-name: Electron app
+name: Electron app BETA
 
 on:
   push:
@@ -11,21 +11,21 @@ jobs:
 
     strategy:
       matrix:
-        os: [ubuntu-18.04, windows-2016]
-        # os: [macOS-10.14, windows-2016, ubuntu-18.04]
+        # os: [ubuntu-18.04, windows-2016]
+        os: [macOS-10.15, windows-2022, ubuntu-18.04]
 
     steps:
       - name: Context
         env:
           GITHUB_CONTEXT: ${{ toJson(github) }}
         run: echo "$GITHUB_CONTEXT"
-      - uses: actions/checkout@v1
+      - uses: actions/checkout@v2
         with:
           fetch-depth: 1
-      - name: Use Node.js 10.x
+      - name: Use Node.js 14.x
         uses: actions/setup-node@v1
         with:
-          node-version: 10.x
+          node-version: 14.x
       - name: yarn install
         run: |
           yarn install
@@ -35,6 +35,9 @@ jobs:
       - name: fillNativeModulesElectron
         run: |
           yarn fillNativeModulesElectron
+      - name: fillPackagedPlugins
+        run: |
+          yarn fillPackagedPlugins
       - name: Install Snapcraft
         if: matrix.os == 'ubuntu-18.04'
         uses: samuelmeuli/action-snapcraft@v1
@@ -63,11 +66,22 @@ jobs:
         run: |
           mkdir artifacts          
 
+          cp app/dist/*.deb artifacts/dbgate-beta.deb || true
+          cp app/dist/*x86*.AppImage artifacts/dbgate-beta.AppImage || true
+          cp app/dist/*arm64*.AppImage artifacts/dbgate-beta-arm64.AppImage || true
+          cp app/dist/*armv7l*.AppImage artifacts/dbgate-beta-armv7l.AppImage || true
+          cp app/dist/*win*.exe artifacts/dbgate-beta.exe || true
+          cp app/dist/*win_x64.zip artifacts/dbgate-windows-beta.zip || true
+          cp app/dist/*win_arm64.zip artifacts/dbgate-windows-beta-arm64.zip || true
+          cp app/dist/*-mac_x64.dmg artifacts/dbgate-beta.dmg || true
+          cp app/dist/*-mac_arm64.dmg artifacts/dbgate-beta-arm64.dmg || true
+
           mv app/dist/*.exe artifacts/ || true
+          mv app/dist/*.zip artifacts/ || true
           mv app/dist/*.AppImage artifacts/ || true
           mv app/dist/*.deb artifacts/ || true
           mv app/dist/*.snap artifacts/ || true
-#          mv app/dist/*.dmg artifacts/ || true
+          mv app/dist/*.dmg artifacts/ || true
 
       - name: Upload artifacts
         uses: actions/upload-artifact@v1

.github/workflows/build-app.yaml

@@ -16,22 +16,24 @@ jobs:
     strategy:
       matrix:
         # os: [ubuntu-18.04, windows-2016]
-        os: [macOS-10.14, windows-2016, ubuntu-18.04]
+        os: [macOS-10.15, windows-2022, ubuntu-18.04]
 
     steps:
       - name: Context
         env:
           GITHUB_CONTEXT: ${{ toJson(github) }}
         run: echo "$GITHUB_CONTEXT"
-      - uses: actions/checkout@v1
+      - uses: actions/checkout@v2
         with:
           fetch-depth: 1
-      - name: Use Node.js 10.x
+      - name: Use Node.js 14.x
         uses: actions/setup-node@v1
         with:
-          node-version: 10.x
+          node-version: 14.x
       - name: yarn install
         run: |
+          # yarn --version
+          # yarn config set network-timeout 300000
           yarn install
       - name: setCurrentVersion
         run: |
@@ -39,6 +41,9 @@ jobs:
       - name: fillNativeModulesElectron
         run: |
           yarn fillNativeModulesElectron
+      - name: fillPackagedPlugins
+        run: |
+          yarn fillPackagedPlugins
       - name: Install Snapcraft
         if: matrix.os == 'ubuntu-18.04'
         uses: samuelmeuli/action-snapcraft@v1
@@ -72,11 +77,17 @@ jobs:
           mkdir artifacts          
 
           cp app/dist/*.deb artifacts/dbgate-latest.deb || true
-          cp app/dist/*.AppImage artifacts/dbgate-latest.AppImage || true
+          cp app/dist/*x86*.AppImage artifacts/dbgate-latest.AppImage || true
+          cp app/dist/*arm64*.AppImage artifacts/dbgate-latest-arm64.AppImage || true
+          cp app/dist/*armv7l*.AppImage artifacts/dbgate-latest-armv7l.AppImage || true
           cp app/dist/*.exe artifacts/dbgate-latest.exe || true
-          cp app/dist/*.dmg artifacts/dbgate-latest.dmg || true
+          cp app/dist/*win_x64.zip artifacts/dbgate-windows-latest.zip || true
+          cp app/dist/*win_arm64.zip artifacts/dbgate-windows-latest-arm64.zip || true
+          cp app/dist/*-mac_x64.dmg artifacts/dbgate-latest.dmg || true
+          cp app/dist/*-mac_arm64.dmg artifacts/dbgate-latest-arm64.dmg || true
 
           mv app/dist/*.exe artifacts/ || true
+          mv app/dist/*.zip artifacts/ || true
           mv app/dist/*.AppImage artifacts/ || true
           mv app/dist/*.deb artifacts/ || true
           mv app/dist/*.dmg artifacts/ || true
@@ -107,7 +118,7 @@ jobs:
       #     mv app/dist/latest.yml artifacts/latest.yml || true
 
       - name: Copy latest.yml (windows)
-        if: matrix.os == 'windows-2016'
+        if: matrix.os == 'windows-2022'
         run: |
           mv app/dist/latest.yml artifacts/latest.yml || true
           mv app/dist/dbgate-pad.xml artifacts/ || true
@@ -118,7 +129,7 @@ jobs:
           mv app/dist/latest-linux.yml artifacts/latest-linux.yml || true
 
       - name: Copy latest-mac.yml
-        if: matrix.os == 'macOS-10.14'
+        if: matrix.os == 'macOS-10.15'
         run: |
           mv app/dist/latest-mac.yml artifacts/latest-mac.yml || true
 

.github/workflows/build-docker-beta.yaml

@@ -0,0 +1,58 @@
+name: Docker image BETA
+
+# on: [push]
+
+on:
+  push:
+    tags:
+      - 'v[0-9]+.[0-9]+.[0-9]+-beta.[0-9]+'
+      - 'v[0-9]+.[0-9]+.[0-9]+-docker.[0-9]+'
+
+jobs:
+  build:
+
+    runs-on: ${{ matrix.os }}
+
+    strategy:
+      matrix:
+        os: [ubuntu-18.04]
+
+    steps:
+      - name: Context
+        env:
+          GITHUB_CONTEXT: ${{ toJson(github) }}
+        run: echo "$GITHUB_CONTEXT"
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 1
+      - name: Use Node.js 14.x
+        uses: actions/setup-node@v1
+        with:
+          node-version: 14.x
+      - name: yarn install
+        run: |
+          # yarn --version
+          # yarn config set network-timeout 300000
+          yarn install
+      - name: setCurrentVersion
+        run: |
+          yarn setCurrentVersion
+      - name: Prepare docker image
+        run: |
+          yarn run prepare:docker
+      - name: Build docker image
+        run: |
+          docker build ./docker -t dbgate
+      - name: Push docker image
+        run: |
+          docker tag dbgate dbgate/dbgate:beta
+          docker login -u ${{ secrets.DOCKER_USERNAME }} -p ${{ secrets.DOCKER_PASSWORD }}
+          docker push dbgate/dbgate:beta
+      - name: Build alpine docker image
+        run: |
+          docker build ./docker -t dbgate -f docker/Dockerfile-alpine
+      - name: Push alpine docker image
+        run: |
+          docker tag dbgate dbgate/dbgate:beta-alpine
+          docker login -u ${{ secrets.DOCKER_USERNAME }} -p ${{ secrets.DOCKER_PASSWORD }}
+          docker push dbgate/dbgate:beta-alpine

.github/workflows/build-docker.yaml

@@ -21,20 +21,19 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-18.04]
-        # os: [macOS-10.14, windows-2016, ubuntu-18.04]
 
     steps:
       - name: Context
         env:
           GITHUB_CONTEXT: ${{ toJson(github) }}
         run: echo "$GITHUB_CONTEXT"
-      - uses: actions/checkout@v1
+      - uses: actions/checkout@v2
         with:
           fetch-depth: 1
-      - name: Use Node.js 10.x
+      - name: Use Node.js 14.x
         uses: actions/setup-node@v1
         with:
-          node-version: 10.x
+          node-version: 14.x
       - name: yarn install
         run: |
           yarn install
@@ -52,3 +51,11 @@ jobs:
           docker tag dbgate dbgate/dbgate
           docker login -u ${{ secrets.DOCKER_USERNAME }} -p ${{ secrets.DOCKER_PASSWORD }}
           docker push dbgate/dbgate
+      - name: Build alpine docker image
+        run: |
+          docker build ./docker -t dbgate -f docker/Dockerfile-alpine
+      - name: Push alpine docker image
+        run: |
+          docker tag dbgate dbgate/dbgate:alpine
+          docker login -u ${{ secrets.DOCKER_USERNAME }} -p ${{ secrets.DOCKER_PASSWORD }}
+          docker push dbgate/dbgate:alpine

.github/workflows/build-npm.yaml

@@ -6,7 +6,7 @@ on:
   push:
     tags:
       - 'v[0-9]+.[0-9]+.[0-9]+'
-      # - 'v[0-9]+.[0-9]+.[0-9]+-alpha.[0-9]+'
+      - 'v[0-9]+.[0-9]+.[0-9]+-alpha.[0-9]+'
 
 # on:
 #   push:
@@ -21,20 +21,19 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-18.04]
-        # os: [macOS-10.14, windows-2016, ubuntu-18.04]
 
     steps:
       - name: Context
         env:
           GITHUB_CONTEXT: ${{ toJson(github) }}
         run: echo "$GITHUB_CONTEXT"
-      - uses: actions/checkout@v1
+      - uses: actions/checkout@v2
         with:
           fetch-depth: 1
-      - name: Use Node.js 10.x
+      - name: Use Node.js 14.x
         uses: actions/setup-node@v1
         with:
-          node-version: 10.x
+          node-version: 14.x
 
       - name: Configure NPM token
         env:
@@ -80,12 +79,67 @@ jobs:
         run: |
           npm publish
 
+      - name: Publish query-splitter
+        working-directory: packages/query-splitter
+        run: |
+          npm publish
+          
       - name: Publish web
         working-directory: packages/web
         run: |
           npm publish
-                                        
-      - name: Publish dbgate
+
+      - name: Publish dbgate (obsolete)
         working-directory: packages/dbgate
         run: |
           npm publish
+
+      - name: Publish dbgate-serve
+        working-directory: packages/serve
+        run: |
+          npm publish
+
+      - name: Publish dbgate-plugin-csv
+        working-directory: plugins/dbgate-plugin-csv
+        run: |
+          npm publish
+
+      - name: Publish dbgate-plugin-xml
+        working-directory: plugins/dbgate-plugin-xml
+        run: |
+          npm publish
+
+      - name: Publish dbgate-plugin-excel
+        working-directory: plugins/dbgate-plugin-excel
+        run: |
+          npm publish
+
+      - name: Publish dbgate-plugin-mssql
+        working-directory: plugins/dbgate-plugin-mssql
+        run: |
+          npm publish
+
+      - name: Publish dbgate-plugin-mysql
+        working-directory: plugins/dbgate-plugin-mysql
+        run: |
+          npm publish
+
+      - name: Publish dbgate-plugin-mongo
+        working-directory: plugins/dbgate-plugin-mongo
+        run: |
+          npm publish
+
+      - name: Publish dbgate-plugin-postgres
+        working-directory: plugins/dbgate-plugin-postgres
+        run: |
+          npm publish
+
+      - name: Publish dbgate-plugin-sqlite
+        working-directory: plugins/dbgate-plugin-sqlite
+        run: |
+          npm publish
+
+      - name: Publish dbgate-plugin-redis
+        working-directory: plugins/dbgate-plugin-redis
+        run: |
+          npm publish

.github/workflows/run-tests.yaml

@@ -0,0 +1,82 @@
+name: Run tests
+on:
+  push:
+    branches:
+      - master
+      - develop
+
+jobs:
+  test-runner:
+    runs-on: ubuntu-latest
+    container: node:14.18
+
+    steps:
+      - name: Context
+        env:
+          GITHUB_CONTEXT: ${{ toJson(github) }}
+        run: echo "$GITHUB_CONTEXT"
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 1
+      - name: yarn install
+        run: |
+          yarn install
+      - name: Integration tests
+        run: |
+          cd integration-tests
+          yarn test:ci
+#          yarn wait:ci
+      - name: Filter parser tests
+        if: always()
+        run: |
+          cd packages/filterparser
+          yarn test:ci
+      - name: Query spliiter tests
+        if: always()
+        run: |
+          cd packages/query-splitter
+          yarn test:ci
+      - uses: tanmen/jest-reporter@v1
+        if: always()
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          result-file: integration-tests/result.json
+          action-name: Integration tests
+      - uses: tanmen/jest-reporter@v1
+        if: always()
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          result-file: packages/filterparser/result.json
+          action-name: Filter parser test results
+      - uses: tanmen/jest-reporter@v1
+        if: always()
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          result-file: packages/query-splitter/result.json
+          action-name: Query splitter test results
+
+    services:
+      postgres:
+        image: postgres
+        env:
+          POSTGRES_PASSWORD: Pwd2020Db
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+
+      mysql:
+        image: mysql:8.0.18
+        env:
+          MYSQL_ROOT_PASSWORD: Pwd2020Db
+
+      mssql:
+        image: mcr.microsoft.com/mssql/server
+        env:
+          ACCEPT_EULA: Y
+          SA_PASSWORD: Pwd2020Db
+          MSSQL_PID: Express
+          
+      # cockroachdb:
+      #   image: cockroachdb/cockroach

.gitignore

@@ -12,6 +12,12 @@ node_modules
 build
 dist
 
+app/packages/web/public
+app/packages/plugins
+docker/public
+docker/bundle.js
+docker/plugins
+
 # misc
 .DS_Store
 .env.local
@@ -24,3 +30,5 @@ yarn-debug.log*
 yarn-error.log*
 app/src/nativeModulesContent.js
 packages/api/src/nativeModulesContent.js
+packages/api/src/packagedPluginsContent.js
+.VSCodeCounter

.vscode/settings.json

@@ -0,0 +1,3 @@
+{
+    "jestrunner.jestCommand": "node_modules/.bin/cross-env DEVMODE=1 LOCALTEST=1 node_modules/.bin/jest"
+}

CHANGELOG.md

@@ -1,5 +1,315 @@
 # ChangeLog
 
+### 4.7.3
+- CHANGED: Export menu redesign, quick export menu merged with old export menu
+- REMOVED: Quick export menu
+- ADDED: Export column mapping
+- ADDED: Export invoked from data grid respects columns choosed in column manager
+- ADDED: Quick export (now merged in export menu) is now possible also in web app
+- FIXED: Virtual foreign key editor fixes
+- FIXED: Tabs panel style fix
+- ADDED: Find by schema in databases widget
+- FIXED: Column manager selection fix
+- FIXED: NPM dist - fixed error when loading plugins
+- CHANGED: NPN dist is now executed by dbgate-serve command
+- ADDED: NPM dist accepts .env configuration
+
+### 4.7.2
+- CHANGED: documentation URL - https://dbgate.org/docs/
+- CHANGED: Close button available for all tab groups - #238
+- ADDED: Search function for the Keyboard Shortcuts overview - #239
+- ADDED: Editor font size settings - #229
+- ADDED: Rename MongoDB collection - #223
+- FIXED: bug in cache subsystem
+
+### 4.7.1
+- FIXED: Fixed connecting to MS SQL server running in docker container from DbGate running in docker container #236 
+- FIXED: Fixed export MongoDB collections into Excel and CSV #240
+- ADDED: Added support for docker volumes to persiste connections, when not using configuration via env variables #232
+- ADDED: DbGate in Docker can run in subdirectory #228
+- FIXED: DbGate in Docker can be proxied with nginx #228
+- FIDED: Theme persists when opening multiple windows #207
+- ADDED: Remember fullscreen state #230
+- ADDED: Improved fullscreen state, title bar with menu is hidden, menu is in hamburger menu, like in web version
+- ADDED: Theme choose dialog (added as tab in settings)
+- FIXED: Fixed crash when clicking on application layers #231
+### 4.7.0
+- CHANGED: Changed main menu style, menu and title bar is in one line (+ability to switch to system menu)
+- REMOVED: Removed main toolbar, use main menu or tab related bottom tool instead
+- ADDED: Added tab related context bottom toolbar
+- ADDED: Main menu is available also in web application, by clicking on hamburger menu
+- ADDED: Added support of SQLite to docker container #219
+- ADDED: Added Debian and Alpine docker distributions (default is Debian)
+- FIXED: Fixed performance problem of data grid, especially when there are cells with large data (eg. JSONs), now it is much faster
+- ADDED: Open JSON and array cell buttons
+- ADDED: Handle JSON in varchar cells
+- ADDED: Scroll tabs on mouse wheel
+- ADDED: Show edit edit MySQL column comments #218 #81
+- ADDED: Handle sparse (mssql), unsigned (mysql), zerofill (mysql) column flags
+- FIXED: Fixed same caching problems (eg. leading to indefinitely loading DB structure sometimes)
+- ADDED: Show estimated table row count for MySQL and MS SQL
+- FIXED: Fixed deleting rows from added rows in table data editor
+- ADDED: Better work with JSON lines file, added JSONL editor with preview
+
+### 4.6.3
+- FIXED: Fixed Windows build
+- FIXED: Fixed crash, when there is invalid value in browser local storage
+- FIXED: Fixed plugin description display, where author name or description is not correctly filled
+
+### 4.6.2
+- FIXED: Fixed issues of XML import plugin
+- ADDED: Split columns macro (available in data sheet editor)
+- CHANGED: Accepting non standard plugins names (which doesn't start with dbgate-plugin-)
+- ADDED: Support BLOB values #211
+- ADDED: Picture cell view
+- ADDED: HTML cell view
+- CHANGED: Code completion supports non-default schema names
+- FIXED: More robust MySQL analyser, when connecting to non-standard servers #214
+- FIXED: Fixed configuring connection to SQLite with environment variables #215
+
+### 4.6.1
+- ADDED: Ability to configure SSH tunnel over environment variables #210 (for docker container)
+- ADDED: XML export and import
+- ADDED: Archive file - show and edit source text file
+- ADDED: Window title shows current tab and database
+- ADDED: DbGate documentation
+- ADDED: Introduced application layers
+- ADDED: Virtual foreign key editor
+- ADDED: Application commands (SQL scripts related to database)
+- ADDED: Theme can be implemented in plugin
+- CHANGED: Dictionary description is stored in app
+- FIXED: Unique and index editor
+- FIXED: Posibility to edit UNIQUE index flag
+- CHANGED: UX improvements of table editor
+
+### 4.6.0
+- ADDED: ER diagrams #118
+    - Generate diagram from table or for database
+    - Automatic layout
+    - Diagram styles - colors, select columns to display, optional displaying data type or nullability
+    - Export diagram to HTML file
+- FIXED: Mac latest build link #204
+
+### 4.5.1
+- FIXED: MongoId detection
+- FIXED: #203 disabled spellchecker
+- FIXED: Prevented display filters in form view twice
+- FIXED: Query designer fixes
+
+### 4.5.0
+- ADDED: #220 functions, materialized views and stored procedures in code completion
+- ADDED: Query result in statusbar
+- ADDED: Highlight and execute current query
+- CHANGED: Code completion offers objects only from current query
+- CHANGED: Big optimalizations of electron app - removed embedded web server, removed remote module, updated electron to version 13
+- CHANGED: Removed dependency to electron-store module
+- FIXED: #201 fixed database URL definition, when running from Docvker container
+- FIXED: #192 Docker container stops in 1 second, ability to stop container with Ctrl+C
+- CHANGED: Web app - websocket replaced with SSE technology
+- CHANGED: Changed tab order, tabs are ordered by creation time
+- ADDED: Reorder tabs with drag & drop
+- CHANGED: Collapse left column in datagrid - removed from settings, remember last used state
+- ADDED: Ability to select multiple columns in column manager in datagrid + copy column names
+- ADDED: Show used filters in left datagrid column
+- FIXED: Fixed delete dependency cycle detection (delete didn't work for some tables)
+
+### 4.4.4
+- FIXED: Database colors
+- CHANGED: Precise work with MongoDB ObjectId
+- FIXED: Run macro works on MongoDB collection data editor
+- ADDED: Type conversion macros
+- CHANGED: Improved UX of import into current database or current archive
+- ADDED: Posibility to create string MongoDB IDs when importing into MongoDB collections
+- CHANGED: Better crash recovery
+- FIXED: Context menu of data editor when using views - some commands didn't work for views
+- ADDED: Widget lists (on left side) now supports add operation, where it has sense
+- CHANGED: Improved UX of saved data sheets
+- ADDED: deploy - preloadedRows: impelemnted onsertOnly columns
+- ADDED: Show change log after app upgrade
+
+### 4.4.3
+- ADDED: Connection and database colors
+- ADDED: Ability to pin connection or table
+- ADDED: MongoDb: create, drop collection from menu
+- ADDED: Copy as MongoDB insert
+- ADDED: MongoDB support for multiple statements in script (dbgate-query-splitter)
+- ADDED: View JSON in tab
+- ADDED: Open DB model as JSON
+- ADDED: Open JSON array as data sheet
+- ADDED: Open JSON from data grid
+- FIXED: Mongo update command when using string IDs resembling Mongo IDs
+- CHANGED: Imrpoved add JSON document, change JSON document commands
+- ADDED: Possibility to add column to JSON grid view
+- FIXED: Hiding columns #1
+- REMOVED: Copy JSON document menu command (please use Copy advanced instead)
+- CHANGED: Save widget visibility and size
+
+### 4.4.2
+- ADDED: Open SQL script from SQL confirm
+- CHANGED: Better looking statusbar
+- ADDED: Create table from database popup menu
+- FIXED: Some fixes for DB compare+deploy (eg. #196)
+- ADDED: Archives + DB models from external directories
+- ADDED: DB deploy supports preloaded data
+- ADDED: Support for Command key on Mac (#199)
+
+### 4.4.1
+- FIXED: #188 Fixed problem with datetime values in PostgreSQL and mysql
+- ADDED: #194 Close tabs by DB
+- FIXED: Improved form view width calculations
+- CHANGED: Form view - highlight matched columns instead of filtering
+- ADDED: Lookup distinct values
+- ADDED: Copy advanced command, Copy as CSV, JSON, YAML, SQL
+- CHANGED: Hide column manager by default
+- ADDED: Change database status command
+- CHANGED: Table structure and view structure tabs have different icons
+- ADDED: #186 - zoom setting
+- ADDED: Row count information moved into status bar, when only one grid on tab is used (typical case)
+
+### 4.4.0
+- ADDED: Database structure compare, export report to HTML
+- ADDED: Experimental: Deploy DB structure changes between databases
+- ADDED: Lookup dialog, available in table view on columns with foreign key
+- ADDED: Customize foreign key lookups
+- ADDED: Chart improvements, export charts as HTML page
+- ADDED: Experimental: work with DB model, deploy model, compare model with real DB
+- ADDED: #193 new SQLite db command
+- CHANGED: #190 code completion improvements
+- ADDED: #189 Copy JSON document - context menu command in data grid for MongoDB
+- ADDED: #191 Connection to POstgreSQL can be defined also with connection string
+- ADDED: #187 dbgate-query-splitter: Transform stream support
+- CHANGED: Upgraded to node 12 in docker app
+- FIXED: Upgraded to node 12 in docker app
+- FIXED: Fixed import into SQLite and PostgreSQL databases, added integration test for this
+
+### 4.3.4
+- FIXED: Delete row with binary ID in MySQL (#182)
+- ADDED: Using 'ODBC Driver 17 for SQL Server' or 'SQL Server Native Client 11.0', when connecting to MS SQL using windows auth #183
+
+### 4.3.3
+- ADDED: Generate SQL from data (#176 - Copy row as INSERT/UPDATE statement)
+- ADDED: Datagrid keyboard column operations (Ctrl+F - find column, Ctrl+H - hide column) #180
+- FIXED: Make window remember that it was maximized
+- FIXED: Fixed lost focus after copy to clipboard and after inserting SQL join
+
+### 4.3.2
+- FIXED: Sorted database list in PostgreSQL (#178)
+- FIXED: Loading stricture of PostgreSQL database, when it contains indexes on expressions (#175)
+- ADDED: Hotkey Shift+Alt+F for formatting SQL code
+
+### 4.3.1
+- FIXED: #173 Using key phrase for SSH key file connection
+- ADDED: #172 Abiloity to quick search within database names
+- ADDED: Database search added to command palette (Ctrl+P)
+- FIXED: #171 fixed PostgreSQL analyser for older versions than 9.3 (matviews don't exist)
+- ADDED: DELETE cascade option - ability to delete all referenced rows, when deleting rows
+
+### 4.3.0
+- ADDED: Table structure editor
+- ADDED: Index support
+- ADDED: Unique constraint support
+- ADDED: Context menu for drop/rename table/columns and for drop view/procedure/function
+- ADDED: Added support for Windows arm64 platform
+- FIXED: Search by _id in MongoDB
+
+### 4.2.6
+- FIXED: Fixed MongoDB import
+- ADDED: Configurable thousands separator #136
+- ADDED: Using case insensitive text search in postgres
+
+### 4.2.5
+- FIXED: Fixed crash when using large model on some installations
+- FIXED: Postgre SQL CREATE function 
+- FIXED: Analysing of MySQL when modifyDate is not known
+
+### 4.2.4
+- ADDED: Query history
+- ADDED: One-click exports in desktop app
+- ADDED: JSON array export
+- FIXED: Procedures in PostgreSQL #122
+- ADDED: Support of materialized views for PostgreSQL #123
+- ADDED: Integration tests
+- FIXED: Fixes in DB structure analysis in PostgreSQL, SQLite, MySQL
+- FIXED: Save data in SQLite, PostgreSQL
+- CHANGED: Introduced package dbgate-query-splitter, instead of sql-query-identifier and @verycrazydog/mysql-parse
+
+### 4.2.3
+- ADDED: ARM builds for MacOS and Linux
+- ADDED: Filter by columns in form view
+
+### 4.2.2
+- CHANGED: Further startup optimalization (approx. 2 times quicker start of electron app)
+
+### 4.2.1
+- FIXED: Fixed+optimalized app startup (esp. on Windows)
+
+### 4.2.0
+- ADDED: Support of SQLite database
+- ADDED: Support of Amazon Redshift database
+- ADDED: Support of CockcroachDB
+- CHANGED: DB Model is not auto-refreshed by default, refresh could be invoked from statusbar
+- FIXED: Fixed race conditions on startup
+- FIXED: Fixed broken style in data grid under strange circumstances
+- ADDED: Configure connections with commandline arguments  #108
+- CHANGED: Optimalized algorithm of incremental DB model updates
+- CHANGED: Loading queries from PostgreSQL doesn't need cursors, using streamed query instead
+- ADDED: Disconnect command
+- ADDED: Query executed on server has tab marker (formerly it had only "No DB" marker)
+- ADDED: Horizontal scroll using shift+mouse wheel #113
+- ADDED: Cosmetic improvements of MariaDB support
+
+### 4.1.11
+- FIX: Fixed crash of API process when using SSH tunnel connection (race condition)
+
+### 4.1.11
+- FIX: fixed processing postgre query containing $$
+- FIX: fixed postgre analysing procedures & functions
+- FIX: patched svelte crash #105
+- ADDED: ability to disbale background DB model updates
+- ADDED: Duplicate connection
+- ADDED: Duplicate tab
+- FIX: SSH tunnel connection using keyfile auth #106
+- FIX: All tables button fix in export #109
+- CHANGED: Add to favorites moved from toolbar to tab context menu
+- CHANGED: Toolbar design - current tab related commands are delimited
+
+### 4.1.10
+- ADDED: Default database option in connectin settings #96 #92
+- FIX: Bundle size optimalization for Windows #97
+- FIX: Popup menu placement on smaller displays #94
+- ADDED: Browse table data with SQL Server 2008 #93
+- FIX: Prevented malicious origins / DNS rebinding #91
+- ADDED: Handle JSON fields in data editor (eg. jsonb field in Postgres) #90
+- FIX: Fixed crash on Windows with Hyper-V #86
+- ADDED: Show database server version in status bar
+- ADDED: Show detailed info about error, when connect to database fails
+- ADDED: Portable ZIP distribution for Windows #84
+### 4.1.9
+- FIX: Incorrect row count info in query result #83
+
+### 4.1.1
+- CHANGED: Default plugins are now part of installation
+### 4.1.0
+- ADDED: MongoDB support
+- ADDED: Configurable keyboard shortcuts
+- ADDED: JSON row cell data view
+- FIX: Fixed some problems from migration to Svelte
+
+### 4.0.3
+- FIX: fixes for FireFox (mainly incorrent handle of bind:clientHeight, replaces with resizeobserver)
+### 4.0.2
+- FIX: fixed docker and NPM build
+### 4.0.0
+- CHANGED: Excahnged React with Svelte. Changed theme colors. Huge speed and memory optimalization
+- ADDED: SQL Generator (CREATE, INSERT, DROP)
+- ADDED: Command palette (F1). Introduced commands, extended some context menus
+- ADDED: New keyboard shortcuts
+- ADDED: Switch to recent database feature
+- ADDED: Macros from free table editor are available also in table data editor
+- CHANGED: Cell data preview is now in left widgets panel
+- CHANGED: Toolbar refactor
+- FIX: Solved reconnecting expired connection
+
 ### 3.9.6
 - ADDED: Connect using SSH Tunnel
 - ADDED: Connect using SSL

README.md

@@ -1,53 +1,103 @@
-[![NPM version](https://img.shields.io/npm/v/dbgate.svg)](https://www.npmjs.com/package/dbgate)
+[![NPM version](https://img.shields.io/npm/v/dbgate-serve.svg)](https://www.npmjs.com/package/dbgate-serve)
+![GitHub All Releases](https://img.shields.io/github/downloads/dbgate/dbgate/total) 
 [![dbgate](https://snapcraft.io/dbgate/badge.svg)](https://snapcraft.io/dbgate)
 [![dbgate](https://snapcraft.io/dbgate/trending.svg?name=0)](https://snapcraft.io/dbgate)
 [![styled with prettier](https://img.shields.io/badge/styled_with-prettier-ff69b4.svg)](https://github.com/prettier/prettier)
-[![Donate](https://img.shields.io/badge/donate-paypal-blue.svg)](https://paypal.me/JanProchazkaCz/30eur)
 
-# DbGate - database administration tool
+<img src="https://raw.githubusercontent.com/dbgate/dbgate/master/app/icon.png" width="64" align="right"/>
 
-DbGate is fast and easy to use database manager. Works with MySQL, PostgreSQL and SQL Server.
+# DbGate - (no)SQL database client
+
+DbGate is cross-platform database manager. 
+It's designed to be simple to use and effective, when working with more databases simultaneously.
+But there are also many advanced features like schema compare, visual query designer, chart visualisation or batch export and import.
+
+DbGate is licensed under MIT license and is completely free.
 
 * Try it online - [demo.dbgate.org](https://demo.dbgate.org) - online demo application
-* Download application for Windows, Linux or Mac from [dbgate.org](https://dbgate.org/download/)
-* Run web version as [NPM package](https://www.npmjs.com/package/dbgate) or as [docker image](https://hub.docker.com/r/dbgate/dbgate)
+* **Download** application for Windows, Linux or Mac from [dbgate.org](https://dbgate.org/download/)
+* Run web version as [NPM package](https://www.npmjs.com/package/dbgate-serve) or as [docker image](https://hub.docker.com/r/dbgate/dbgate)
 
-![Screenshot](https://raw.githubusercontent.com/dbgate/dbgate/master/screenshot.png)
+## Supported databases
+* MySQL
+* PostgreSQL
+* SQL Server
+* MongoDB
+* SQLite
+* Amazon Redshift
+* CockroachDB
+* MariaDB
+
+<!-- Learn more about DbGate features at the [DbGate website](https://dbgate.org/), or try our online [demo application](https://demo.dbgate.org) -->
+
+
+<a href="https://raw.githubusercontent.com/dbgate/dbgate/master/img/screenshot1.png">
+    <img src="https://raw.githubusercontent.com/dbgate/dbgate/master/img/screenshot1.png" width="400"/>
+</a>
+<a href="https://raw.githubusercontent.com/dbgate/dbgate/master/img/screenshot2.png">
+    <img src="https://raw.githubusercontent.com/dbgate/dbgate/master/img/screenshot2.png" width="400"/>
+</a>
+<a href="https://raw.githubusercontent.com/dbgate/dbgate/master/img/screenshot4.png">
+    <img src="https://raw.githubusercontent.com/dbgate/dbgate/master/img/screenshot4.png" width="400"/>
+</a>
+<a href="https://raw.githubusercontent.com/dbgate/dbgate/master/img/screenshot3.png">
+    <img src="https://raw.githubusercontent.com/dbgate/dbgate/master/img/screenshot3.png" width="400"/>
+</a>
+
+<!-- ![Screenshot](https://raw.githubusercontent.com/dbgate/dbgate/master/screenshot.png) -->
 
 ## Features
-* Connect to Microsoft SQL Server, Postgre SQL, MySQL
 * Table data editing, with SQL change script preview
-* Master/detail views
+* Edit table schema, indexes, primary and foreign keys
+* Compare and synchronize database structure
+* ER diagram
+* Light and dark theme
+* Master/detail views, foreign key lookups
 * Query designer
 * Form view for comfortable work with tables with many columns
-* Explore tables, views, procedures, functions
-* SQL editor, execute SQL script, SQL code formatter, SQL code completion, SQL join wizard
+* JSON view on MongoDB collections
+* Explore tables, views, procedures, functions, MongoDB collections
+* SQL editor
+  * execute SQL script
+  * SQL code formatter
+  * SQL code completion
+  * Add SQL LEFT/INNER/RIGHT join utility
+* Mongo JavaScript editor, execute Mongo script (with NodeJs syntax)
 * Runs as application for Windows, Linux and Mac. Or in Docker container on server and in web Browser on client.
-* Import, export from/to CSV, Excel, JSON
+* Import, export from/to CSV, Excel, JSON, XML
 * Free table editor - quick table data editing (cleanup data after import/before export, prototype tables etc.)
 * Archives - backup your data in JSON files on local filesystem (or on DbGate server, when using web application)
-* Light and dark theme
-* Charts
+* Charts, export chart to HTML page
 * For detailed info, how to run DbGate in docker container, visit [docker hub](https://hub.docker.com/r/dbgate/dbgate)
 * Extensible plugin architecture
 
+## How to contribute
+Any contributions are welcome. If you want to contribute without coding, consider following:
+
+* Tell your friends about DbGate or share on social networks - when more people will use DbGate, it will grow to be better
+* Write review on [Slant.co](https://www.slant.co/improve/options/41086/~dbgate-review) or [G2](https://www.g2.com/products/dbgate/reviews) 
+* Create issue, if you find problem in app, or you have idea to new feature. If issue already exists, you could leave comment on it, to prioritise most wanted issues.
+* Become a backer on [Open collective](https://opencollective.com/dbgate)
+* Where a small coding is acceptable for you, you could [create plugin](https://dbgate.org/docs/plugin-development.html). Plugins for new themes can be created actually without JS coding.
+
+Thank you!
+
 ## Why is DbGate different
 There are many database managers now, so why DbGate? 
 * Works everywhere - Windows, Linux, Mac, Web browser (+mobile web is planned), without compromises in features
 * Based on standalone NPM packages, scripts can be run without DbGate (example - [CSV export](https://www.npmjs.com/package/dbgate-plugin-csv) )
-* Many data browsing functions based using foreign keys - master/detail, expand columns, expandable form view (on screenshot above)
+* Many data browsing functions based using foreign keys - master/detail, expand columns, expandable form view
 
 ## Design goals
 * Application simplicity - DbGate takes the best and only the best from old [DbGate](http://www.jenasoft.com/dbgate), [DatAdmin](http://www.jenasoft.com/datadmin) and [DbMouse](http://www.jenasoft.com/dbmouse) .
 * Minimal dependencies
-    * Frontend - React, styled-components, socket.io
+    * Frontend - Svelte, socket.io
     * Backend - NodeJs, ExpressJs, socket.io, database connection drivers
     * JavaScript + TypeScript
     * App - electron
-    * There is plan to incorporate SQLite to support work with local datasets
-* Platform independed - will run as web application in single docker container on server, or as application using Electron platform on Linux, Windows and Mac
+* Platform independent - runs as web application in single docker container on server, or as application using Electron platform on Linux, Windows and Mac
 
-## Plugins
+<!-- ## Plugins
 Plugins are standard NPM packages published on [npmjs.com](https://www.npmjs.com).  
 See all [existing DbGate plugins](https://www.npmjs.com/search?q=keywords:dbgateplugin).  
 Visit [dbgate generator homepage](https://github.com/dbgate/generator-dbgate) to see, how to create your own plugin.  
@@ -56,29 +106,46 @@ Currently following extensions can be implemented using plugins:
 - File format parsers/writers
 - Database engine connectors
 
+Basic set of plugins is part of DbGate git repository and is installed with app. Additional plugins pust be downloaded from NPM (this task is handled by DbGate) -->
+
 ## How to run development environment
 
+Simple variant - runs WEB application:
 ```sh
 yarn
 yarn start
 ```
 
-If you want to make modifications in TypeScript packages, run TypeScript compiler in watch mode in seconds terminal:
+If you want more control, run WEB application:
 ```sh
-yarn lib
+yarn # install NPM packages
 ```
 
+And than run following 3 commands concurrently in 3 terminals:
+```
+yarn start:api # run API on port 3000
+yarn start:web # run web on port 5000
+yarn lib # watch typescript libraries and plugins modifications
+```
+This runs API on port 3000 and web application on port 5000  
 Open http://localhost:5000 in your browser
 
-You could run electron app (requires running localhost:5000):
+If you want to run electron app:
 ```sh
+yarn # install NPM packages
 cd app
-yarn
-yarn start
+yarn # install NPM packages for electron
+```
+
+And than run following 3 commands concurrently in 3 terminals:
+```
+yarn start:web # run web on port 5000 (only static JS and HTML files)
+yarn lib # watch typescript libraries and plugins modifications
+yarn start:app # run electron app
 ```
 
 ## How to run built electron app locally
-This mode is very similar to production run of electron app. Electron app forks process with API on dynamically allocated port, works with compiled javascript files (doesn't use localhost:5000)
+This mode is very similar to production run of electron app. Electron doesn't use localhost:5000.
 
 ```sh
 cd app
@@ -91,15 +158,16 @@ yarn build:app:local
 yarn start:app:local
 ```
 
-## Packages
-Some dbgate packages can be used also without DbGate. You can find them on [NPM repository](https://www.npmjs.com/search?q=keywords:dbgate)
+## How to create plugin
+Creating plugin is described in [documentation](https://github.com/dbgate/dbgate/wiki/Plugin-development)
 
-* [api](https://github.com/dbgate/dbgate/tree/master/packages/api) - backend, Javascript, ExpressJS [![NPM version](https://img.shields.io/npm/v/dbgate-api.svg)](https://www.npmjs.com/package/dbgate-api)
-* [datalib](https://github.com/dbgate/dbgate/tree/master/packages/datalib) - TypeScript library for utility classes [![NPM version](https://img.shields.io/npm/v/dbgate-datalib.svg)](https://www.npmjs.com/package/dbgate-datalib)
-* [app](https://github.com/dbgate/dbgate/tree/master/app) - application (JavaScript) structure, creating specific queries (JavaScript)
-* [filterparser](https://github.com/dbgate/dbgate/tree/master/packages/filterparser) - TypeScript library for parsing data filter expressions using parsimmon [![NPM version](https://img.shields.io/npm/v/dbgate-filterparser.svg)](https://www.npmjs.com/package/dbgate-filterparser)
-* [sqltree](https://github.com/dbgate/dbgate/tree/master/packages/sqltree) - JSON representation of SQL query, functions converting to SQL (TypeScript) [![NPM version](https://img.shields.io/npm/v/dbgate-sqltree.svg)](https://www.npmjs.com/package/dbgate-sqltree)
-* [types](https://github.com/dbgate/dbgate/tree/master/packages/types) - common TypeScript definitions [![NPM version](https://img.shields.io/npm/v/dbgate-types.svg)](https://www.npmjs.com/package/dbgate-types)
-* [web](https://github.com/dbgate/dbgate/tree/master/packages/web) - frontend in React (JavaScript) [![NPM version](https://img.shields.io/npm/v/dbgate-web.svg)](https://www.npmjs.com/package/dbgate-web)
-* [tools](https://github.com/dbgate/dbgate/tree/master/packages/tools) - various tools [![NPM version](https://img.shields.io/npm/v/dbgate-tools.svg)](https://www.npmjs.com/package/dbgate-tools)
+But it is very simple:
 
+```sh
+npm install -g yo # install yeoman
+npm install -g generator-dbgate # install dbgate generator
+cd dbgate-plugin-my-new-plugin # this directory is created by wizard, edit, what you need to change
+yarn plugin # this compiles plugin and copies it into existing DbGate installation
+```
+
+After restarting DbGate, you could use your new plugin from DbGate.

app/package.json

@@ -1,41 +1,59 @@
 {
   "name": "dbgate",
-  "version": "3.9.5",
+  "version": "4.1.1",
   "private": true,
   "author": "Jan Prochazka <jenasoft.database@gmail.com>",
   "description": "Opensource database administration tool",
   "dependencies": {
-    "electron-log": "^4.3.1",
-    "electron-store": "^5.1.1",
-    "electron-updater": "^4.3.5"
+    "electron-log": "^4.4.1",
+    "electron-updater": "^4.6.1",
+    "lodash.clonedeepwith": "^4.5.0",
+    "patch-package": "^6.4.7"
   },
   "repository": {
     "type": "git",
     "url": "https://github.com/dbgate/dbgate.git"
   },
   "build": {
+    "artifactName": "${productName}-${version}-${os}_${arch}.${ext}",
     "appId": "org.dbgate",
     "mac": {
       "category": "database",
       "icon": "icon512.png",
-      "artifactName": "dbgate-mac-${version}.${ext}",
+      "publish": [
+        "github"
+      ],
+      "target": {
+        "target": "default",
+        "arch": [
+          "arm64",
+          "x64"
+        ]
+      }
+    },
+    "linux": {
+      "target": [
+        "deb",
+        "snap",
+        {
+          "target": "AppImage",
+          "arch": [
+            "x64",
+            "armv7l",
+            "arm64"
+          ]
+        }
+      ],
+      "icon": "icons/",
+      "category": "Development",
+      "synopsis": "Database manager for SQL Server, MySQL, PostgreSQL, MongoDB and SQLite",
       "publish": [
         "github"
       ]
     },
-    "linux": {
-      "target": [
-        "AppImage",
-        "deb",
-        "snap"
-      ],
-      "icon": "icon.png",
-      "artifactName": "dbgate-linux-${version}.${ext}",
-      "category": "Development",
-      "synopsis": "Database administration tool for MS SQL, MySQL and PostgreSQL",
-      "publish": [
-        "github"
-      ]
+    "appImage": {
+      "license": "./LICENSE",
+      "category": "Development"
     },
     "snap": {
       "publish": [
@@ -48,9 +66,15 @@
     },
     "win": {
       "target": [
-        "nsis"
+        "nsis",
+        {
+          "target": "zip",
+          "arch": [
+            "x64",
+            "arm64"
+          ]
+        }
       ],
-      "artifactName": "dbgate-windows-${version}.${ext}",
       "icon": "icon.ico",
       "publish": [
         "github"
@@ -65,22 +89,24 @@
   },
   "homepage": "./",
   "scripts": {
-    "start": "cross-env ELECTRON_START_URL=http://localhost:5000 electron .",
+    "start": "cross-env ELECTRON_START_URL=http://localhost:5000 DEVMODE=1 electron .",
     "start:local": "cross-env electron .",
     "dist": "electron-builder",
-    "build": "cd ../packages/api && yarn build && cd ../web && yarn build:app && cd ../../app && yarn dist",
-    "build:local": "cd ../packages/api && yarn build && cd ../web && yarn build:app && cd ../../app && yarn predist",
-    "postinstall": "electron-builder install-app-deps",
-    "predist": "copyfiles ../packages/api/dist/* packages && copyfiles \"../packages/web/build/*\" packages && copyfiles \"../packages/web/build/**/*\" packages"
+    "build": "cd ../packages/api && yarn build && cd ../web && yarn build && cd ../../app && yarn dist",
+    "build:local": "cd ../packages/api && yarn build && cd ../web && yarn build && cd ../../app && yarn predist",
+    "postinstall": "yarn rebuild && patch-package",
+    "rebuild": "electron-builder install-app-deps",
+    "predist": "copyfiles ../packages/api/dist/* packages && copyfiles \"../packages/web/public/*\" packages && copyfiles \"../packages/web/public/**/*\" packages && copyfiles --up 3 \"../plugins/dist/**/*\" packages/plugins"
   },
   "main": "src/electron.js",
   "devDependencies": {
     "copyfiles": "^2.2.0",
     "cross-env": "^6.0.3",
-    "electron": "11.2.3",
-    "electron-builder": "22.9.1"
+    "electron": "13.6.3",
+    "electron-builder": "22.14.5"
   },
   "optionalDependencies": {
-    "msnodesqlv8": "^2.0.10"
+    "better-sqlite3": "7.4.5",
+    "msnodesqlv8": "^2.4.4"
   }
-}
+}

app/src/electron.js

@@ -1,10 +1,10 @@
 const electron = require('electron');
 const os = require('os');
+const fs = require('fs');
 const { Menu, ipcMain } = require('electron');
-const { fork } = require('child_process');
 const { autoUpdater } = require('electron-updater');
-const Store = require('electron-store');
 const log = require('electron-log');
+const _cloneDeepWith = require('lodash.clonedeepwith');
 
 // Module to control application life.
 const app = electron.app;
@@ -13,13 +13,33 @@ const BrowserWindow = electron.BrowserWindow;
 
 const path = require('path');
 const url = require('url');
+const mainMenuDefinition = require('./mainMenuDefinition');
+const { settings } = require('cluster');
 
-const store = new Store();
+// require('@electron/remote/main').initialize();
+
+const configRootPath = path.join(app.getPath('userData'), 'config-root.json');
+let initialConfig = {};
+
+try {
+  initialConfig = JSON.parse(fs.readFileSync(configRootPath, { encoding: 'utf-8' }));
+} catch (err) {
+  console.log('Error loading config-root:', err.message);
+  initialConfig = {};
+}
+
+// let settingsJson = {};
+// try {
+//   const datadir = path.join(os.homedir(), 'dbgate-data');
+//   settingsJson = JSON.parse(fs.readFileSync(path.join(datadir, 'settings.json'), { encoding: 'utf-8' }));
+// } catch (err) {
+//   console.log('Error loading settings.json:', err.message);
+//   settingsJson = {};
+// }
 
 // Keep a global reference of the window object, if you don't, the window will
 // be closed automatically when the JavaScript object is garbage collected.
 let mainWindow;
-let splashWindow;
 let mainMenu;
 
 log.transports.file.level = 'debug';
@@ -27,158 +47,163 @@ autoUpdater.logger = log;
 // TODO - create settings for this
 // appUpdater.channel = 'beta';
 
-function hideSplash() {
-  if (splashWindow) {
-    splashWindow.destroy();
-    splashWindow = null;
-  }
-  mainWindow.show();
+let commands = {};
+
+function commandItem(id) {
+  const command = commands[id];
+  return {
+    id,
+    label: command ? command.menuName || command.toolbarName || command.name : id,
+    accelerator: command ? command.keyText : undefined,
+    enabled: command ? command.enabled : false,
+    click() {
+      mainWindow.webContents.send('run-command', id);
+    },
+  };
 }
 
 function buildMenu() {
-  const template = [
-    {
-      label: 'File',
-      submenu: [
-        {
-          label: 'Connect to database',
-          click() {
-            mainWindow.webContents.executeJavaScript(`dbgate_createNewConnection()`);
-          },
-        },
-        {
-          label: 'Open file',
-          click() {
-            mainWindow.webContents.executeJavaScript(`dbgate_openFile()`);
-          },
-        },
-        {
-          label: 'Save',
-          click() {
-            mainWindow.webContents.executeJavaScript(`dbgate_tabCommand('save')`);
-          },
-          accelerator: 'Ctrl+S',
-          id: 'save',
-        },
-        {
-          label: 'Save As',
-          click() {
-            mainWindow.webContents.executeJavaScript(`dbgate_tabCommand('saveAs')`);
-          },
-          accelerator: 'Ctrl+Shift+S',
-          id: 'saveAs',
-        },
-        { type: 'separator' },
-        { role: 'close' },
-      ],
-    },
-    {
-      label: 'Window',
-      submenu: [
-        {
-          label: 'New query',
-          click() {
-            mainWindow.webContents.executeJavaScript(`dbgate_newQuery()`);
-          },
-        },
-        { type: 'separator' },
-        {
-          label: 'Close all tabs',
-          click() {
-            mainWindow.webContents.executeJavaScript('dbgate_closeAll()');
-          },
-        },
-        { role: 'minimize' },
-      ],
-    },
+  const template = _cloneDeepWith(mainMenuDefinition, item => {
+    if (item.divider) {
+      return { type: 'separator' };
+    }
 
-    // {
-    //   label: 'Edit',
-    //   submenu: [
-    //     { role: 'undo' },
-    //     { role: 'redo' },
-    //     { type: 'separator' },
-    //     { role: 'cut' },
-    //     { role: 'copy' },
-    //     { role: 'paste' },
-    //   ],
-    // },
-    {
-      label: 'View',
-      submenu: [
-        { role: 'reload' },
-        { role: 'forcereload' },
-        { role: 'toggledevtools' },
-        { type: 'separator' },
-        { role: 'resetzoom' },
-        { role: 'zoomin' },
-        { role: 'zoomout' },
-        { type: 'separator' },
-        { role: 'togglefullscreen' },
-      ],
-    },
-    {
-      role: 'help',
-      submenu: [
-        {
-          label: 'dbgate.org',
-          click() {
-            require('electron').shell.openExternal('https://dbgate.org');
-          },
-        },
-        {
-          label: 'DbGate on GitHub',
-          click() {
-            require('electron').shell.openExternal('https://github.com/dbgate/dbgate');
-          },
-        },
-        {
-          label: 'DbGate on docker hub',
-          click() {
-            require('electron').shell.openExternal('https://hub.docker.com/r/dbgate/dbgate');
-          },
-        },
-        {
-          label: 'Report problem or feature request',
-          click() {
-            require('electron').shell.openExternal('https://github.com/dbgate/dbgate/issues/new');
-          },
-        },
-        {
-          label: 'About',
-          click() {
-            mainWindow.webContents.executeJavaScript(`dbgate_showAbout()`);
-          },
-        },
-      ],
-    },
-  ];
+    if (item.command) {
+      return commandItem(item.command);
+    }
+  });
 
   return Menu.buildFromTemplate(template);
 }
 
-ipcMain.on('update-menu', async (event, arg) => {
-  const commands = await mainWindow.webContents.executeJavaScript(`dbgate_getCurrentTabCommands()`);
-  mainMenu.getMenuItemById('save').enabled = !!commands.save;
-  mainMenu.getMenuItemById('saveAs').enabled = !!commands.saveAs;
+ipcMain.on('update-commands', async (event, arg) => {
+  commands = JSON.parse(arg);
+  for (const key of Object.keys(commands)) {
+    const menu = mainMenu.getMenuItemById(key);
+    if (!menu) continue;
+    const command = commands[key];
+
+    // rebuild menu
+    if (menu.label != command.text || menu.accelerator != command.keyText) {
+      mainMenu = buildMenu();
+      mainWindow.setMenu(mainMenu);
+      return;
+    }
+
+    menu.enabled = command.enabled;
+  }
+});
+ipcMain.on('close-window', async (event, arg) => {
+  mainWindow.close();
+});
+ipcMain.on('set-title', async (event, arg) => {
+  mainWindow.setTitle(arg);
+});
+ipcMain.on('open-link', async (event, arg) => {
+  electron.shell.openExternal(arg);
+});
+ipcMain.on('window-action', async (event, arg) => {
+  switch (arg) {
+    case 'minimize':
+      mainWindow.minimize();
+      break;
+    case 'maximize':
+      if (mainWindow.isMaximized()) {
+        mainWindow.unmaximize();
+      } else {
+        mainWindow.maximize();
+      }
+      break;
+    case 'close':
+      mainWindow.close();
+      break;
+    case 'fullscreen-on':
+      mainWindow.setFullScreen(true);
+      break;
+    case 'fullscreen-off':
+      mainWindow.setFullScreen(false);
+      break;
+    case 'devtools':
+      mainWindow.webContents.toggleDevTools();
+      break;
+    case 'reload':
+      mainWindow.webContents.reloadIgnoringCache();
+      break;
+    case 'zoomin':
+      mainWindow.webContents.zoomLevel += 0.5;
+      break;
+    case 'zoomout':
+      mainWindow.webContents.zoomLevel -= 0.5;
+      break;
+    case 'zoomreset':
+      mainWindow.webContents.zoomLevel = 0;
+      break;
+  }
 });
 
+ipcMain.handle('showOpenDialog', async (event, options) => {
+  const res = electron.dialog.showOpenDialogSync(mainWindow, options);
+  return res;
+});
+ipcMain.handle('showSaveDialog', async (event, options) => {
+  const res = electron.dialog.showSaveDialogSync(mainWindow, options);
+  return res;
+});
+ipcMain.handle('showItemInFolder', async (event, path) => {
+  electron.shell.showItemInFolder(path);
+});
+ipcMain.handle('openExternal', async (event, url) => {
+  electron.shell.openExternal(url);
+});
+
+function fillMissingSettings(value) {
+  const res = {
+    ...value,
+  };
+  if (value['app.useNativeMenu'] !== true && value['app.useNativeMenu'] !== false) {
+    res['app.useNativeMenu'] = os.platform() == 'darwin' ? true : false;
+  }
+  return res;
+}
+
 function createWindow() {
-  const bounds = store.get('winBounds');
+  let settingsJson = {};
+  try {
+    const datadir = path.join(os.homedir(), 'dbgate-data');
+    settingsJson = fillMissingSettings(
+      JSON.parse(fs.readFileSync(path.join(datadir, 'settings.json'), { encoding: 'utf-8' }))
+    );
+  } catch (err) {
+    console.log('Error loading settings.json:', err.message);
+    settingsJson = fillMissingSettings({});
+  }
+
+  const bounds = initialConfig['winBounds'];
+  useNativeMenu = settingsJson['app.useNativeMenu'];
 
   mainWindow = new BrowserWindow({
     width: 1200,
     height: 800,
     title: 'DbGate',
+    frame: useNativeMenu,
+    titleBarStyle: useNativeMenu ? undefined : 'hidden',
     ...bounds,
     icon: os.platform() == 'win32' ? 'icon.ico' : path.resolve(__dirname, '../icon.png'),
-    show: false,
+    partition: 'persist:dbgate',
     webPreferences: {
       nodeIntegration: true,
-      enableRemoteModule: true,
+      contextIsolation: false,
+      spellcheck: false,
     },
   });
 
+  if (initialConfig['winIsMaximized']) {
+    mainWindow.maximize();
+  }
+  if (settingsJson['app.fullscreen']) {
+    mainWindow.setFullScreen(true);
+  }
+
   mainMenu = buildMenu();
   mainWindow.setMenu(mainMenu);
 
@@ -186,59 +211,52 @@ function createWindow() {
     const startUrl =
       process.env.ELECTRON_START_URL ||
       url.format({
-        pathname: path.join(__dirname, '../packages/web/build/index.html'),
+        pathname: path.join(__dirname, '../packages/web/public/index.html'),
         protocol: 'file:',
         slashes: true,
       });
-    mainWindow.webContents.on('did-finish-load', function () {
-      hideSplash();
-    });
     mainWindow.on('close', () => {
-      store.set('winBounds', mainWindow.getBounds());
+      try {
+        fs.writeFileSync(
+          configRootPath,
+          JSON.stringify({
+            winBounds: mainWindow.getBounds(),
+            winIsMaximized: mainWindow.isMaximized(),
+          }),
+          'utf-8'
+        );
+      } catch (err) {
+        console.log('Error saving config-root:', err.message);
+      }
     });
     mainWindow.loadURL(startUrl);
     if (os.platform() == 'linux') {
       mainWindow.setIcon(path.resolve(__dirname, '../icon.png'));
     }
+    // mainWindow.webContents.toggleDevTools();
   }
 
-  splashWindow = new BrowserWindow({
-    width: 300,
-    height: 120,
-    transparent: true,
-    frame: false,
-  });
-  splashWindow.loadURL(
-    url.format({
-      pathname: path.join(__dirname, '../packages/web/build/splash.html'),
-      protocol: 'file:',
-      slashes: true,
-    })
+  const apiPackage = path.join(
+    __dirname,
+    process.env.DEVMODE ? '../../packages/api/src/index' : '../packages/api/dist/bundle.js'
   );
 
-  if (process.env.ELECTRON_START_URL) {
-    loadMainWindow();
-  } else {
-    const apiProcess = fork(path.join(__dirname, '../packages/api/dist/bundle.js'), [
-      '--dynport',
-      '--native-modules',
-      path.join(__dirname, 'nativeModules'),
-      // '../../../src/nativeModules'
-    ]);
-    apiProcess.on('message', msg => {
-      if (msg.msgtype == 'listening') {
-        const { port } = msg;
-        global['port'] = port;
-        loadMainWindow();
-      }
-    });
-  }
+  global.API_PACKAGE = apiPackage;
+  global.NATIVE_MODULES = path.join(__dirname, 'nativeModules');
 
-  // and load the index.html of the app.
-  // mainWindow.loadURL('http://localhost:3000');
+  // console.log('global.API_PACKAGE', global.API_PACKAGE);
+  const api = require(apiPackage);
+  // console.log(
+  //   'REQUIRED',
+  //   path.resolve(
+  //     path.join(__dirname, process.env.DEVMODE ? '../../packages/api/src/index' : '../packages/api/dist/bundle.js')
+  //   )
+  // );
+  const main = api.getMainModule();
+  main.initializeElectronSender(mainWindow.webContents);
+  main.useAllControllers(null, electron);
 
-  // Open the DevTools.
-  // mainWindow.webContents.openDevTools();
+  loadMainWindow();
 
   // Emitted when the window is closed.
   mainWindow.on('closed', function () {
@@ -250,7 +268,9 @@ function createWindow() {
 }
 
 function onAppReady() {
-  autoUpdater.checkForUpdatesAndNotify();
+  if (!process.env.DEVMODE) {
+    autoUpdater.checkForUpdatesAndNotify();
+  }
   createWindow();
 }
 

app/src/mainMenuDefinition.js

@@ -0,0 +1,83 @@
+module.exports = [
+  {
+    label: 'File',
+    submenu: [
+      { command: 'new.connection', hideDisabled: true },
+      { command: 'new.sqliteDatabase', hideDisabled: true },
+      { divider: true },
+      { command: 'new.query', hideDisabled: true },
+      { command: 'new.freetable', hideDisabled: true },
+      { command: 'new.shell', hideDisabled: true },
+      { command: 'new.jsonl', hideDisabled: true },
+      { divider: true },
+      { command: 'file.open', hideDisabled: true },
+      { command: 'file.openArchive', hideDisabled: true },
+      { divider: true },
+      { command: 'group.save', hideDisabled: true },
+      { command: 'group.saveAs', hideDisabled: true },
+      { divider: true },
+      { command: 'file.exit', hideDisabled: true },
+      { command: 'app.logout', hideDisabled: true },
+    ],
+  },
+  {
+    label: 'Window',
+    submenu: [
+      { command: 'tabs.closeTab', hideDisabled: false },
+      { command: 'tabs.closeAll', hideDisabled: false },
+      { command: 'tabs.closeTabsWithCurrentDb', hideDisabled: false },
+      { command: 'tabs.closeTabsButCurrentDb', hideDisabled: false },
+      { divider: true },
+      { command: 'app.zoomIn', hideDisabled: true },
+      { command: 'app.zoomOut', hideDisabled: true },
+      { command: 'app.zoomReset', hideDisabled: true },
+    ],
+  },
+
+  // {
+  //   label: 'Edit',
+  //   submenu: [
+  //     { role: 'undo' },
+  //     { role: 'redo' },
+  //     { type: 'separator' },
+  //     { role: 'cut' },
+  //     { role: 'copy' },
+  //     { role: 'paste' },
+  //   ],
+  // },
+  {
+    label: 'View',
+    submenu: [
+      { command: 'app.reload', hideDisabled: true },
+      { command: 'app.toggleDevTools', hideDisabled: true },
+      { command: 'app.toggleFullScreen', hideDisabled: true },
+      { command: 'app.minimize', hideDisabled: true },
+      { divider: true },
+      { command: 'theme.changeTheme', hideDisabled: true },
+      { command: 'settings.show' },
+    ],
+  },
+  {
+    label: 'Tools',
+    submenu: [
+      { command: 'database.search', hideDisabled: true },
+      { command: 'commandPalette.show', hideDisabled: true },
+      { divider: true },
+      { command: 'sql.generator', hideDisabled: true },
+      { command: 'file.import', hideDisabled: true },
+      { command: 'new.modelCompare', hideDisabled: true },
+    ],
+  },
+  {
+    label: 'Help',
+    submenu: [
+      { command: 'app.openDocs', hideDisabled: true },
+      { command: 'app.openWeb', hideDisabled: true },
+      { command: 'app.openIssue', hideDisabled: true },
+      { command: 'app.openSponsoring', hideDisabled: true },
+      { divider: true },
+      { command: 'tabs.changelog', hideDisabled: true },
+      { command: 'about.show', hideDisabled: true },
+    ],
+  },
+];

docker-compose.yaml

@@ -0,0 +1,54 @@
+# this compose file is for testing purposes only
+# use it for testing docker containsers built on local machine
+version: "3"
+services:
+  dbgate:
+    build: docker
+    # image: dbgate/dbgate:beta-alpine
+    # image: dbgate/dbgate:alpine
+    # image: dbgate/dbgate:beta
+    restart: always
+    ports:
+      - 3100:3000
+    # volumes:
+    #   - /home/jena/dbgate-data:/root/dbgate-data
+
+    volumes:
+      - dbgate-data:/root/dbgate-data
+    
+    # environment:
+    #   WEB_ROOT: /dbgate
+
+      # CONNECTIONS: mssql
+      # LABEL_mssql: MS Sql
+      # SERVER_mssql: mssql
+      # USER_mssql: sa
+      # PORT_mssql: 1433
+      # PASSWORD_mssql: Pwd2020Db
+      # ENGINE_mssql: mssql@dbgate-plugin-mssql
+  # proxy:
+  #   # image: nginx
+  #   build: test/nginx
+  #   ports:
+  #     - 8082:80
+
+    # volumes:
+    #   - /home/jena/test/chinook:/mnt/sqt
+    # environment:
+    #   CONNECTIONS: sqlite
+
+    #   LABEL_sqlite: sqt
+    #   FILE_sqlite: /mnt/sqt/Chinook.db
+    #   ENGINE_sqlite: sqlite@dbgate-plugin-sqlite
+
+  # mssql:
+  #   image: mcr.microsoft.com/mssql/server
+  #   restart: always
+  #   environment:
+  #     - ACCEPT_EULA=Y
+  #     - SA_PASSWORD=Pwd2020Db
+  #     - MSSQL_PID=Express
+
+volumes:
+  dbgate-data:
+    driver: local

docker/.gitignore

@@ -0,0 +1,2 @@
+package.json
+yarn.lock

docker/Dockerfile

@@ -1,4 +1,4 @@
-FROM node:12-alpine
+FROM node:14
 
 WORKDIR /home/dbgate-docker
 
@@ -6,4 +6,5 @@ COPY . .
 
 WORKDIR /home/dbgate-docker
 EXPOSE 3000
+VOLUME /root/dbgate-data
 CMD node bundle.js

docker/Dockerfile-alpine

@@ -0,0 +1,10 @@
+FROM node:14-alpine
+
+WORKDIR /home/dbgate-docker
+
+COPY . .
+
+WORKDIR /home/dbgate-docker
+EXPOSE 3000
+VOLUME /root/dbgate-data
+CMD node bundle.js

fillNativeModules.js

@@ -2,10 +2,10 @@ const fs = require('fs');
 
 let fillContent = '';
 
-// if (!process.argv.includes('--electron')) {
 if (process.platform == 'win32') {
   fillContent += `content.msnodesqlv8 = () => require('msnodesqlv8');`;
 }
+fillContent += `content['better-sqlite3'] = () => require('better-sqlite3');`;
 
 const getContent = (empty) => `
 // this file is generated automatically by script fillNativeModules.js, do not edit it manually

fillPackagedPlugins.js

@@ -0,0 +1,23 @@
+const fs = require('fs');
+const path = require('path');
+
+function load() {
+  const plugins = {};
+
+  for (const packageName of fs.readdirSync('plugins')) {
+    if (!packageName.startsWith('dbgate-plugin-')) continue;
+    const dir = path.join('plugins', packageName);
+    const frontend = fs.readFileSync(path.join(dir, 'dist', 'frontend.js'), 'utf-8');
+    const readme = fs.readFileSync(path.join(dir, 'README.md'), 'utf-8');
+    const manifest = JSON.parse(fs.readFileSync(path.join(dir, 'package.json'), 'utf-8'));
+    plugins[packageName] = {
+      manifest,
+      frontend,
+      readme,
+    };
+  }
+
+  return plugins;
+}
+
+fs.writeFileSync('packages/api/src/packagedPluginsContent.js', `module.exports = () => (${JSON.stringify(load())});`);

integration-tests/.gitignore

@@ -0,0 +1 @@
+dbtemp

integration-tests/__tests__/alter-database.spec.js

@@ -0,0 +1,68 @@
+const stableStringify = require('json-stable-stringify');
+const _ = require('lodash');
+const fp = require('lodash/fp');
+const uuidv1 = require('uuid/v1');
+const { testWrapper } = require('../tools');
+const engines = require('../engines');
+const { getAlterDatabaseScript, extendDatabaseInfo, generateDbPairingId } = require('dbgate-tools');
+
+function flatSource() {
+  return _.flatten(
+    engines.map(engine => (engine.objects || []).map(object => [engine.label, object.type, object, engine]))
+  );
+}
+
+async function testDatabaseDiff(conn, driver, mangle, createObject = null) {
+  await driver.query(conn, `create table t1 (id int not null primary key)`);
+
+  await driver.query(
+    conn,
+    `create table t2 (
+    id int not null primary key, 
+    t1_id int null references t1(id)
+  )`
+  );
+
+  if (createObject) await driver.query(conn, createObject);
+
+  const structure1 = generateDbPairingId(extendDatabaseInfo(await driver.analyseFull(conn)));
+  let structure2 = _.cloneDeep(structure1);
+  mangle(structure2);
+  structure2 = extendDatabaseInfo(structure2);
+
+  const { sql } = getAlterDatabaseScript(structure1, structure2, {}, structure1, structure2, driver);
+  console.log('RUNNING ALTER SQL', driver.engine, ':', sql);
+
+  await driver.script(conn, sql);
+
+  const structure2Real = extendDatabaseInfo(await driver.analyseFull(conn));
+
+  expect(structure2Real.tables.length).toEqual(structure2.tables.length);
+  return structure2Real;
+}
+
+describe('Alter database', () => {
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Drop referenced table - %s',
+    testWrapper(async (conn, driver, engine) => {
+      await testDatabaseDiff(conn, driver, db => {
+        _.remove(db.tables, x => x.pureName == 't1');
+      });
+    })
+  );
+
+  test.each(flatSource())(
+    'Drop object - %s - %s',
+    testWrapper(async (conn, driver, type, object, engine) => {
+      const db = await testDatabaseDiff(
+        conn,
+        driver,
+        db => {
+          _.remove(db[type], x => x.pureName == 'obj1');
+        },
+        object.create1
+      );
+      expect(db[type].length).toEqual(0);
+    })
+  );
+});

integration-tests/__tests__/alter-table.spec.js

@@ -0,0 +1,124 @@
+const stableStringify = require('json-stable-stringify');
+const _ = require('lodash');
+const fp = require('lodash/fp');
+const uuidv1 = require('uuid/v1');
+const { testWrapper } = require('../tools');
+const engines = require('../engines');
+const { getAlterTableScript, extendDatabaseInfo, generateDbPairingId } = require('dbgate-tools');
+
+function pickImportantTableInfo(table) {
+  return {
+    pureName: table.pureName,
+    columns: table.columns
+      .filter(x => x.columnName != 'rowid')
+      .map(fp.pick(['columnName', 'notNull', 'autoIncrement'])),
+  };
+}
+
+function checkTableStructure(t1, t2) {
+  // expect(t1.pureName).toEqual(t2.pureName)
+  expect(pickImportantTableInfo(t1)).toEqual(pickImportantTableInfo(t2));
+}
+
+async function testTableDiff(conn, driver, mangle) {
+  await driver.query(conn, `create table t0 (id int not null primary key)`);
+
+  await driver.query(
+    conn,
+    `create table t1 (
+    col_pk int not null primary key, 
+    col_std int null, 
+    col_def int null default 12,
+    col_fk int null references t0(id),
+    col_idx int null,
+    col_uq int null unique,
+    col_ref int null unique
+  )`
+  );
+
+  await driver.query(conn, `create index idx1 on t1(col_idx)`);
+
+  await driver.query(conn, `create table t2 (id int not null primary key, fkval int null references t1(col_ref))`);
+
+  const tget = x => x.tables.find(y => y.pureName == 't1');
+  const structure1 = generateDbPairingId(extendDatabaseInfo(await driver.analyseFull(conn)));
+  let structure2 = _.cloneDeep(structure1);
+  mangle(tget(structure2));
+  structure2 = extendDatabaseInfo(structure2);
+
+  const { sql } = getAlterTableScript(tget(structure1), tget(structure2), {}, structure1, structure2, driver);
+  console.log('RUNNING ALTER SQL', driver.engine, ':', sql);
+
+  await driver.script(conn, sql);
+
+  const structure2Real = extendDatabaseInfo(await driver.analyseFull(conn));
+
+  checkTableStructure(tget(structure2Real), tget(structure2));
+  // expect(stableStringify(structure2)).toEqual(stableStringify(structure2Real));
+}
+
+const TESTED_COLUMNS = ['col_pk', 'col_std', 'col_def', 'col_fk', 'col_ref', 'col_idx', 'col_uq'];
+// const TESTED_COLUMNS = ['col_pk'];
+// const TESTED_COLUMNS = ['col_idx'];
+// const TESTED_COLUMNS = ['col_def'];
+// const TESTED_COLUMNS = ['col_std'];
+// const TESTED_COLUMNS = ['col_ref'];
+
+function engines_columns_source() {
+  return _.flatten(engines.map(engine => TESTED_COLUMNS.map(column => [engine.label, column, engine])));
+}
+
+describe('Alter table', () => {
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Add column - %s',
+    testWrapper(async (conn, driver, engine) => {
+      await testTableDiff(conn, driver, tbl => {
+        tbl.columns.push({
+          columnName: 'added',
+          dataType: 'int',
+          pairingId: uuidv1(),
+          notNull: false,
+          autoIncrement: false,
+        });
+      });
+    })
+  );
+
+  test.each(engines_columns_source())(
+    'Drop column - %s - %s',
+    testWrapper(async (conn, driver, column, engine) => {
+      await testTableDiff(conn, driver, tbl => (tbl.columns = tbl.columns.filter(x => x.columnName != column)));
+    })
+  );
+
+  test.each(engines_columns_source())(
+    'Change nullability - %s - %s',
+    testWrapper(async (conn, driver, column, engine) => {
+      await testTableDiff(
+        conn,
+        driver,
+        tbl => (tbl.columns = tbl.columns.map(x => (x.columnName == column ? { ...x, notNull: true } : x)))
+      );
+    })
+  );
+
+  test.each(engines_columns_source())(
+    'Rename column - %s - %s',
+    testWrapper(async (conn, driver, column, engine) => {
+      await testTableDiff(
+        conn,
+        driver,
+        tbl => (tbl.columns = tbl.columns.map(x => (x.columnName == column ? { ...x, columnName: 'col_renamed' } : x)))
+      );
+    })
+  );
+
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Drop index - %s',
+    testWrapper(async (conn, driver, engine) => {
+      await testTableDiff(conn, driver, tbl => {
+        tbl.indexes = [];
+      });
+    })
+  );
+});

integration-tests/__tests__/db-import.spec.js

@@ -0,0 +1,75 @@
+const engines = require('../engines');
+const stream = require('stream');
+const { testWrapper } = require('../tools');
+const tableWriter = require('dbgate-api/src/shell/tableWriter');
+const copyStream = require('dbgate-api/src/shell/copyStream');
+const fakeObjectReader = require('dbgate-api/src/shell/fakeObjectReader');
+
+function createImportStream() {
+  const pass = new stream.PassThrough({
+    objectMode: true,
+  });
+  pass.write({ columns: [{ columnName: 'id' }, { columnName: 'country' }], __isStreamHeader: true });
+  pass.write({ id: 1, country: 'Czechia' });
+  pass.write({ id: 2, country: 'Austria' });
+  pass.write({ country: 'Germany', id: 3 });
+  pass.write({ country: 'Romania', id: 4 });
+  pass.write({ country: 'Great Britain', id: 5 });
+  pass.write({ country: 'Bosna, Hecegovina', id: 6 });
+  pass.end();
+
+  return pass;
+}
+
+describe('DB Import', () => {
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Import one table - %s',
+    testWrapper(async (conn, driver, engine) => {
+      // const reader = await fakeObjectReader({ delay: 10 });
+      // const reader = await fakeObjectReader();
+      const reader = createImportStream();
+      const writer = await tableWriter({
+        systemConnection: conn,
+        driver,
+        pureName: 't1',
+        createIfNotExists: true,
+      });
+      await copyStream(reader, writer);
+
+      const res = await driver.query(conn, `select count(*) as cnt from t1`);
+      expect(res.rows[0].cnt.toString()).toEqual('6');
+    })
+  );
+
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Import two tables - %s',
+    testWrapper(async (conn, driver, engine) => {
+      // const reader = await fakeObjectReader({ delay: 10 });
+      // const reader = await fakeObjectReader();
+      const reader1 = createImportStream();
+      const writer1 = await tableWriter({
+        systemConnection: conn,
+        driver,
+        pureName: 't1',
+        createIfNotExists: true,
+      });
+      await copyStream(reader1, writer1);
+
+      const reader2 = createImportStream();
+      const writer2 = await tableWriter({
+        systemConnection: conn,
+        driver,
+        pureName: 't2',
+        createIfNotExists: true,
+      });
+      await copyStream(reader2, writer2);
+
+      const res1 = await driver.query(conn, `select count(*) as cnt from t1`);
+      expect(res1.rows[0].cnt.toString()).toEqual('6');
+
+      const res2 = await driver.query(conn, `select count(*) as cnt from t2`);
+      expect(res2.rows[0].cnt.toString()).toEqual('6');
+    })
+  );
+
+});

integration-tests/__tests__/deploy-database.spec.js

@@ -0,0 +1,300 @@
+/// TODO
+
+const { testWrapper } = require('../tools');
+const _ = require('lodash');
+const engines = require('../engines');
+const deployDb = require('dbgate-api/src/shell/deployDb');
+const { databaseInfoFromYamlModel } = require('dbgate-tools');
+const generateDeploySql = require('dbgate-api/src/shell/generateDeploySql');
+
+function checkStructure(structure, model) {
+  const expected = databaseInfoFromYamlModel(model);
+  expect(structure.tables.length).toEqual(expected.tables.length);
+
+  for (const [realTable, expectedTable] of _.zip(
+    _.sortBy(structure.tables, 'pureName'),
+    _.sortBy(expected.tables, 'pureName')
+  )) {
+    expect(realTable.columns.length).toBeGreaterThanOrEqual(expectedTable.columns.length);
+  }
+}
+
+async function testDatabaseDeploy(conn, driver, dbModelsYaml, testEmptyLastScript) {
+  let index = 0;
+  for (const loadedDbModel of dbModelsYaml) {
+    const { sql, isEmpty } = await generateDeploySql({
+      systemConnection: conn,
+      driver,
+      loadedDbModel,
+    });
+    console.debug('Generated deploy script:', sql);
+    expect(sql.toUpperCase().includes('DROP ')).toBeFalsy();
+
+    console.log('dbModelsYaml.length', dbModelsYaml.length, index);
+    if (testEmptyLastScript && index == dbModelsYaml.length - 1) {
+      expect(isEmpty).toBeTruthy();
+    }
+
+    await deployDb({
+      systemConnection: conn,
+      driver,
+      loadedDbModel,
+    });
+
+    index++;
+  }
+
+  const structure = await driver.analyseFull(conn);
+  checkStructure(structure, dbModelsYaml[dbModelsYaml.length - 1]);
+}
+
+describe('Deploy database', () => {
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Deploy database simple - %s',
+    testWrapper(async (conn, driver, engine) => {
+      await testDatabaseDeploy(conn, driver, [
+        [
+          {
+            name: 't1.table.yaml',
+            json: {
+              name: 't1',
+              columns: [{ name: 'id', type: 'int' }],
+              primaryKey: ['id'],
+            },
+          },
+        ],
+      ]);
+    })
+  );
+
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Deploy database simple twice - %s',
+    testWrapper(async (conn, driver, engine) => {
+      await testDatabaseDeploy(
+        conn,
+        driver,
+        [
+          [
+            {
+              name: 't1.table.yaml',
+              json: {
+                name: 't1',
+                columns: [{ name: 'id', type: 'int' }],
+                primaryKey: ['id'],
+              },
+            },
+          ],
+          [
+            {
+              name: 't1.table.yaml',
+              json: {
+                name: 't1',
+                columns: [{ name: 'id', type: 'int' }],
+                primaryKey: ['id'],
+              },
+            },
+          ],
+        ],
+        true
+      );
+    })
+  );
+
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Add column - %s',
+    testWrapper(async (conn, driver, engine) => {
+      await testDatabaseDeploy(conn, driver, [
+        [
+          {
+            name: 't1.table.yaml',
+            json: {
+              name: 't1',
+              columns: [{ name: 'id', type: 'int' }],
+              primaryKey: ['id'],
+            },
+          },
+        ],
+        [
+          {
+            name: 't1.table.yaml',
+            json: {
+              name: 't1',
+              columns: [
+                { name: 'id', type: 'int' },
+                { name: 'val', type: 'int' },
+              ],
+              primaryKey: ['id'],
+            },
+          },
+        ],
+      ]);
+    })
+  );
+
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Dont drop column - %s',
+    testWrapper(async (conn, driver, engine) => {
+      await testDatabaseDeploy(
+        conn,
+        driver,
+        [
+          [
+            {
+              name: 't1.table.yaml',
+              json: {
+                name: 't1',
+                columns: [
+                  { name: 'id', type: 'int' },
+                  { name: 'val', type: 'int' },
+                ],
+                primaryKey: ['id'],
+              },
+            },
+          ],
+          [
+            {
+              name: 't1.table.yaml',
+              json: {
+                name: 't1',
+                columns: [{ name: 'id', type: 'int' }],
+                primaryKey: ['id'],
+              },
+            },
+          ],
+        ],
+        true
+      );
+    })
+  );
+
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Foreign keys - %s',
+    testWrapper(async (conn, driver, engine) => {
+      await testDatabaseDeploy(
+        conn,
+        driver,
+        [
+          [
+            {
+              name: 't2.table.yaml',
+              json: {
+                name: 't2',
+                columns: [
+                  { name: 't2id', type: 'int' },
+                  { name: 't1id', type: 'int', references: 't1' },
+                ],
+                primaryKey: ['t2id'],
+              },
+            },
+            {
+              name: 't1.table.yaml',
+              json: {
+                name: 't1',
+                columns: [{ name: 't1id', type: 'int' }],
+                primaryKey: ['t1id'],
+              },
+            },
+          ],
+          [
+            {
+              name: 't2.table.yaml',
+              json: {
+                name: 't2',
+                columns: [
+                  { name: 't2id', type: 'int' },
+                  { name: 't1id', type: 'int', references: 't1' },
+                ],
+                primaryKey: ['t2id'],
+              },
+            },
+            {
+              name: 't1.table.yaml',
+              json: {
+                name: 't1',
+                columns: [{ name: 't1id', type: 'int' }],
+                primaryKey: ['t1id'],
+              },
+            },
+          ],
+        ],
+        true
+      );
+    })
+  );
+
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Deploy preloaded data - %s',
+    testWrapper(async (conn, driver, engine) => {
+      await testDatabaseDeploy(conn, driver, [
+        [
+          {
+            name: 't1.table.yaml',
+            json: {
+              name: 't1',
+              columns: [
+                { name: 'id', type: 'int' },
+                { name: 'value', type: 'int' },
+              ],
+              primaryKey: ['id'],
+              data: [
+                { id: 1, value: 1 },
+                { id: 2, value: 2 },
+                { id: 3, value: 3 },
+              ],
+            },
+          },
+        ],
+      ]);
+
+      const res = await driver.query(conn, `select count(*) as cnt from t1`);
+      expect(res.rows[0].cnt.toString()).toEqual('3');
+    })
+  );
+
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Deploy preloaded data - update - %s',
+    testWrapper(async (conn, driver, engine) => {
+      await testDatabaseDeploy(conn, driver, [
+        [
+          {
+            name: 't1.table.yaml',
+            json: {
+              name: 't1',
+              columns: [
+                { name: 'id', type: 'int' },
+                { name: 'val', type: 'int' },
+              ],
+              primaryKey: ['id'],
+              data: [
+                { id: 1, val: 1 },
+                { id: 2, val: 2 },
+                { id: 3, val: 3 },
+              ],
+            },
+          },
+        ],
+        [
+          {
+            name: 't1.table.yaml',
+            json: {
+              name: 't1',
+              columns: [
+                { name: 'id', type: 'int' },
+                { name: 'val', type: 'int' },
+              ],
+              primaryKey: ['id'],
+              data: [
+                { id: 1, val: 1 },
+                { id: 2, val: 5 },
+                { id: 3, val: 3 },
+              ],
+            },
+          },
+        ],
+      ]);
+
+      const res = await driver.query(conn, `select val from t1 where id = 2`);
+      expect(res.rows[0].val.toString()).toEqual('5');
+    })
+  );
+});

integration-tests/__tests__/object-analyse.spec.js

@@ -0,0 +1,89 @@
+const { testWrapper } = require('../tools');
+const engines = require('../engines');
+const _ = require('lodash');
+
+const initSql = ['CREATE TABLE t1 (id int)', 'CREATE TABLE t2 (id int)'];
+
+function flatSource() {
+  return _.flatten(
+    engines.map(engine => (engine.objects || []).map(object => [engine.label, object.type, object, engine]))
+  );
+}
+
+const obj1Match = expect.objectContaining({
+  pureName: 'obj1',
+});
+const view1Match = expect.objectContaining({
+  pureName: 'obj1',
+  columns: expect.arrayContaining([
+    expect.objectContaining({
+      columnName: 'id',
+    }),
+  ]),
+});
+
+describe('Object analyse', () => {
+  test.each(flatSource())(
+    'Full analysis - %s - %s',
+    testWrapper(async (conn, driver, type, object, engine) => {
+      for (const sql of initSql) await driver.query(conn, sql);
+
+      await driver.query(conn, object.create1);
+      const structure = await driver.analyseFull(conn);
+
+      expect(structure[type].length).toEqual(1);
+      expect(structure[type][0]).toEqual(type.includes('views') ? view1Match : obj1Match);
+    })
+  );
+
+  test.each(flatSource())(
+    'Incremental analysis - add - %s - %s',
+    testWrapper(async (conn, driver, type, object, engine) => {
+      for (const sql of initSql) await driver.query(conn, sql);
+
+      await driver.query(conn, object.create2);
+      const structure1 = await driver.analyseFull(conn);
+      await driver.query(conn, object.create1);
+      const structure2 = await driver.analyseIncremental(conn, structure1);
+
+      expect(structure2[type].length).toEqual(2);
+      expect(structure2[type].find(x => x.pureName == 'obj1')).toEqual(type.includes('views') ? view1Match : obj1Match);
+    })
+  );
+
+  test.each(flatSource())(
+    'Incremental analysis - drop - %s - %s',
+    testWrapper(async (conn, driver, type, object, engine) => {
+      for (const sql of initSql) await driver.query(conn, sql);
+
+      await driver.query(conn, object.create1);
+      await driver.query(conn, object.create2);
+      const structure1 = await driver.analyseFull(conn);
+      await driver.query(conn, object.drop2);
+      const structure2 = await driver.analyseIncremental(conn, structure1);
+
+      expect(structure2[type].length).toEqual(1);
+      expect(structure2[type][0]).toEqual(type.includes('views') ? view1Match : obj1Match);
+    })
+  );
+
+  test.each(flatSource())(
+    'Create SQL - add - %s - %s',
+    testWrapper(async (conn, driver, type, object, engine) => {
+      for (const sql of initSql) await driver.query(conn, sql);
+
+      await driver.query(conn, object.create1);
+      const structure1 = await driver.analyseFull(conn);
+      await driver.query(conn, object.drop1);
+      const structure2 = await driver.analyseIncremental(conn, structure1);
+      expect(structure2[type].length).toEqual(0);
+
+      await driver.query(conn, structure1[type][0].createSql);
+
+      const structure3 = await driver.analyseIncremental(conn, structure2);
+
+      expect(structure3[type].length).toEqual(1);
+      expect(structure3[type][0]).toEqual(type.includes('views') ? view1Match : obj1Match);
+    })
+  );
+});

integration-tests/__tests__/query.spec.js

@@ -0,0 +1,163 @@
+const engines = require('../engines');
+const { splitQuery } = require('dbgate-query-splitter');
+const { testWrapper } = require('../tools');
+
+const initSql = ['CREATE TABLE t1 (id int)', 'INSERT INTO t1 (id) VALUES (1)', 'INSERT INTO t1 (id) VALUES (2)'];
+
+expect.extend({
+  dataRow(row, expected) {
+    for (const key in expected) {
+      if (row[key] != expected[key]) {
+        return {
+          pass: false,
+          message: () => `Different key: ${key}`,
+        };
+      }
+    }
+    return {
+      pass: true,
+      message: () => '',
+    };
+  },
+});
+
+class StreamHandler {
+  constructor(resolve) {
+    this.results = [];
+    this.resolve = resolve;
+    this.infoRows = [];
+  }
+  row(row) {
+    this.results[this.results.length - 1].rows.push(row);
+  }
+  recordset(columns) {
+    this.results.push({
+      columns,
+      rows: [],
+    });
+  }
+  done(result) {
+    this.resolve(this.results);
+  }
+  info(msg) {
+    this.infoRows.push(msg);
+  }
+}
+
+function executeStreamItem(driver, conn, sql) {
+  return new Promise(resolve => {
+    const handler = new StreamHandler(resolve);
+    driver.stream(conn, sql, handler);
+  });
+}
+
+async function executeStream(driver, conn, sql) {
+  const results = [];
+  for (const sqlItem of splitQuery(sql, driver.getQuerySplitterOptions('stream'))) {
+    const item = await executeStreamItem(driver, conn, sqlItem);
+    results.push(...item);
+  }
+  return results;
+}
+
+describe('Query', () => {
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Simple query - %s',
+    testWrapper(async (conn, driver, engine) => {
+      for (const sql of initSql) await driver.query(conn, sql);
+
+      const res = await driver.query(conn, 'SELECT id FROM t1 ORDER BY id');
+      expect(res.columns).toEqual([
+        expect.objectContaining({
+          columnName: 'id',
+        }),
+      ]);
+
+      expect(res.rows).toEqual([
+        expect.dataRow({
+          id: 1,
+        }),
+        expect.dataRow({
+          id: 2,
+        }),
+      ]);
+    })
+  );
+
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Simple stream query - %s',
+    testWrapper(async (conn, driver, engine) => {
+      for (const sql of initSql) await driver.query(conn, sql);
+      const results = await executeStream(driver, conn, 'SELECT id FROM t1 ORDER BY id');
+      expect(results.length).toEqual(1);
+      const res = results[0];
+
+      expect(res.columns).toEqual([expect.objectContaining({ columnName: 'id' })]);
+      expect(res.rows).toEqual([expect.dataRow({ id: 1 }), expect.dataRow({ id: 2 })]);
+    })
+  );
+
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'More queries - %s',
+    testWrapper(async (conn, driver, engine) => {
+      for (const sql of initSql) await driver.query(conn, sql);
+      const results = await executeStream(
+        driver,
+        conn,
+        'SELECT id FROM t1 ORDER BY id; SELECT id FROM t1 ORDER BY id DESC'
+      );
+      expect(results.length).toEqual(2);
+
+      const res1 = results[0];
+      expect(res1.columns).toEqual([expect.objectContaining({ columnName: 'id' })]);
+      expect(res1.rows).toEqual([expect.dataRow({ id: 1 }), expect.dataRow({ id: 2 })]);
+
+      const res2 = results[1];
+      expect(res2.columns).toEqual([expect.objectContaining({ columnName: 'id' })]);
+      expect(res2.rows).toEqual([expect.dataRow({ id: 2 }), expect.dataRow({ id: 1 })]);
+    })
+  );
+
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Script - return data - %s',
+    testWrapper(async (conn, driver, engine) => {
+      const results = await executeStream(
+        driver,
+        conn,
+        'CREATE TABLE t1 (id int); INSERT INTO t1 (id) VALUES (1); INSERT INTO t1 (id) VALUES (2); SELECT id FROM t1 ORDER BY id; '
+      );
+      expect(results.length).toEqual(1);
+
+      const res1 = results[0];
+      expect(res1.columns).toEqual([expect.objectContaining({ columnName: 'id' })]);
+      expect(res1.rows).toEqual([expect.dataRow({ id: 1 }), expect.dataRow({ id: 2 })]);
+    })
+  );
+
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Script - no data - %s',
+    testWrapper(async (conn, driver, engine) => {
+      const results = await executeStream(
+        driver,
+        conn,
+        'CREATE TABLE t1 (id int); INSERT INTO t1 (id) VALUES (1); INSERT INTO t1 (id) VALUES (2) '
+      );
+      expect(results.length).toEqual(0);
+    })
+  );
+
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Save data query - %s',
+    testWrapper(async (conn, driver, engine) => {
+      for (const sql of initSql) await driver.query(conn, sql);
+
+      await driver.script(
+        conn,
+        'INSERT INTO t1 (id) VALUES (3);INSERT INTO t1 (id) VALUES (4);UPDATE t1 SET id=10 WHERE id=1;DELETE FROM t1 WHERE id=2;'
+      );
+      const res = await driver.query(conn, 'SELECT COUNT(*) AS cnt FROM t1');
+      // console.log(res);
+      expect(res.rows[0].cnt == 3).toBeTruthy();
+    })
+  );
+});

integration-tests/__tests__/table-analyse.spec.js

@@ -0,0 +1,164 @@
+const engines = require('../engines');
+const { testWrapper } = require('../tools');
+
+const t1Sql = 'CREATE TABLE t1 (id int not null primary key, val1 varchar(50) null)';
+const ix1Sql = 'CREATE index ix1 ON t1(val1, id)';
+const t2Sql = 'CREATE TABLE t2 (id int not null primary key, val2 varchar(50) null unique)';
+const t3Sql = 'CREATE TABLE t3 (id int not null primary key, valfk int, foreign key (valfk) references t2(id))';
+// const fkSql = 'ALTER TABLE t3 ADD FOREIGN KEY (valfk) REFERENCES t2(id)'
+
+const txMatch = (tname, vcolname, nextcol) =>
+  expect.objectContaining({
+    pureName: tname,
+    columns: [
+      expect.objectContaining({
+        columnName: 'id',
+        notNull: true,
+        dataType: expect.stringContaining('int'),
+      }),
+      expect.objectContaining({
+        columnName: vcolname,
+        notNull: false,
+        dataType: expect.stringMatching(/.*char.*\(50\)/),
+      }),
+      ...(nextcol
+        ? [
+            expect.objectContaining({
+              columnName: 'nextcol',
+              notNull: false,
+              dataType: expect.stringMatching(/.*char.*\(50\)/),
+            }),
+          ]
+        : []),
+    ],
+    primaryKey: expect.objectContaining({
+      columns: [
+        expect.objectContaining({
+          columnName: 'id',
+        }),
+      ],
+    }),
+  });
+
+const t1Match = txMatch('t1', 'val1');
+const t2Match = txMatch('t2', 'val2');
+const t2NextColMatch = txMatch('t2', 'val2', true);
+
+describe('Table analyse', () => {
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Table structure - full analysis - %s',
+    testWrapper(async (conn, driver, engine) => {
+      await driver.query(conn, t1Sql);
+
+      const structure = await driver.analyseFull(conn);
+
+      expect(structure.tables.length).toEqual(1);
+      expect(structure.tables[0]).toEqual(t1Match);
+    })
+  );
+
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Table add - incremental analysis - %s',
+    testWrapper(async (conn, driver, engine) => {
+      await driver.query(conn, t2Sql);
+
+      const structure1 = await driver.analyseFull(conn);
+      expect(structure1.tables.length).toEqual(1);
+      expect(structure1.tables[0]).toEqual(t2Match);
+
+      await driver.query(conn, t1Sql);
+      const structure2 = await driver.analyseIncremental(conn, structure1);
+
+      expect(structure2.tables.length).toEqual(2);
+      expect(structure2.tables.find(x => x.pureName == 't1')).toEqual(t1Match);
+      expect(structure2.tables.find(x => x.pureName == 't2')).toEqual(t2Match);
+    })
+  );
+
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Table remove - incremental analysis - %s',
+    testWrapper(async (conn, driver, engine) => {
+      await driver.query(conn, t1Sql);
+      await driver.query(conn, t2Sql);
+      const structure1 = await driver.analyseFull(conn);
+      expect(structure1.tables.length).toEqual(2);
+      expect(structure1.tables.find(x => x.pureName == 't1')).toEqual(t1Match);
+      expect(structure1.tables.find(x => x.pureName == 't2')).toEqual(t2Match);
+
+      await driver.query(conn, 'DROP TABLE t2');
+      const structure2 = await driver.analyseIncremental(conn, structure1);
+
+      expect(structure2.tables.length).toEqual(1);
+      expect(structure2.tables[0]).toEqual(t1Match);
+    })
+  );
+
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Table change - incremental analysis - %s',
+    testWrapper(async (conn, driver, engine) => {
+      await driver.query(conn, t1Sql);
+      await driver.query(conn, t2Sql);
+      const structure1 = await driver.analyseFull(conn);
+
+      if (engine.dbSnapshotBySeconds) await new Promise(resolve => setTimeout(resolve, 1100));
+
+      await driver.query(conn, 'ALTER TABLE t2 ADD nextcol varchar(50)');
+      const structure2 = await driver.analyseIncremental(conn, structure1);
+
+      expect(structure2).toBeTruthy(); // if falsy, no modification is detected
+
+      expect(structure2.tables.length).toEqual(2);
+      expect(structure2.tables.find(x => x.pureName == 't1')).toEqual(t1Match);
+      expect(structure2.tables.find(x => x.pureName == 't2')).toEqual(t2NextColMatch);
+    })
+  );
+
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Index - full analysis - %s',
+    testWrapper(async (conn, driver, engine) => {
+      await driver.query(conn, t1Sql);
+      await driver.query(conn, ix1Sql);
+      const structure = await driver.analyseFull(conn);
+
+      const t1 = structure.tables.find(x => x.pureName == 't1');
+      expect(t1.indexes.length).toEqual(1);
+      expect(t1.indexes[0].columns.length).toEqual(2);
+      expect(t1.indexes[0].columns[0]).toEqual(expect.objectContaining({ columnName: 'val1' }));
+      expect(t1.indexes[0].columns[1]).toEqual(expect.objectContaining({ columnName: 'id' }));
+    })
+  );
+
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Unique - full analysis - %s',
+    testWrapper(async (conn, driver, engine) => {
+      await driver.query(conn, t2Sql);
+      const structure = await driver.analyseFull(conn);
+
+      const t2 = structure.tables.find(x => x.pureName == 't2');
+      // const indexesAndUniques = [...t2.uniques, ...t2.indexes];
+      expect(t2.uniques.length).toEqual(1);
+      expect(t2.uniques[0].columns.length).toEqual(1);
+      expect(t2.uniques[0].columns[0]).toEqual(expect.objectContaining({ columnName: 'val2' }));
+    })
+  );
+
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Foreign key - full analysis - %s',
+    testWrapper(async (conn, driver, engine) => {
+      await driver.query(conn, t2Sql);
+      await driver.query(conn, t3Sql);
+      // await driver.query(conn, fkSql);
+
+      const structure = await driver.analyseFull(conn);
+
+      const t3 = structure.tables.find(x => x.pureName == 't3');
+      console.log('T3', t3.foreignKeys[0].columns);
+      expect(t3.foreignKeys.length).toEqual(1);
+      expect(t3.foreignKeys[0].columns.length).toEqual(1);
+      expect(t3.foreignKeys[0]).toEqual(expect.objectContaining({ refTableName: 't2' }));
+      expect(t3.foreignKeys[0].columns[0]).toEqual(
+        expect.objectContaining({ columnName: 'valfk', refColumnName: 'id' })
+      );
+    })
+  );
+});

integration-tests/__tests__/table-create.spec.js

@@ -0,0 +1,153 @@
+const _ = require('lodash');
+const fp = require('lodash/fp');
+const engines = require('../engines');
+const { testWrapper } = require('../tools');
+const { extendDatabaseInfo } = require('dbgate-tools');
+
+function createExpector(value) {
+  return _.cloneDeepWith(value, x => {
+    if (_.isPlainObject(x)) {
+      return expect.objectContaining(_.mapValues(x, y => createExpector(y)));
+    }
+  });
+}
+
+function omitTableSpecificInfo(table) {
+  return {
+    ...table,
+    columns: table.columns.map(fp.omit(['dataType'])),
+  };
+}
+
+function checkTableStructure2(t1, t2) {
+  // expect(t1.pureName).toEqual(t2.pureName)
+  expect(t2).toEqual(createExpector(omitTableSpecificInfo(t1)));
+}
+
+async function testTableCreate(conn, driver, table) {
+  await driver.query(conn, `create table t0 (id int not null primary key)`);
+
+  const dmp = driver.createDumper();
+  const table1 = {
+    ...table,
+    pureName: 'tested',
+  };
+  dmp.createTable(table1);
+
+  console.log('RUNNING CREATE SQL', driver.engine, ':', dmp.s);
+  await driver.script(conn, dmp.s);
+
+  const db = extendDatabaseInfo(await driver.analyseFull(conn));
+  const table2 = db.tables.find(x => x.pureName == 'tested');
+
+  checkTableStructure2(table1, table2);
+}
+
+describe('Table create', () => {
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Simple table - %s',
+    testWrapper(async (conn, driver, engine) => {
+      await testTableCreate(conn, driver, {
+        columns: [
+          {
+            columnName: 'col1',
+            dataType: 'int',
+            notNull: true,
+          },
+        ],
+        primaryKey: {
+          columns: [{ columnName: 'col1' }],
+        },
+      });
+    })
+  );
+
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Table with index - %s',
+    testWrapper(async (conn, driver, engine) => {
+      await testTableCreate(conn, driver, {
+        columns: [
+          {
+            columnName: 'col1',
+            dataType: 'int',
+            notNull: true,
+          },
+          {
+            columnName: 'col2',
+            dataType: 'int',
+            notNull: true,
+          },
+        ],
+        primaryKey: {
+          columns: [{ columnName: 'col1' }],
+        },
+        indexes: [
+          {
+            constraintName: 'ix1',
+            pureName: 'tested',
+            columns: [{ columnName: 'col2' }],
+          },
+        ],
+      });
+    })
+  );
+
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Table with foreign key - %s',
+    testWrapper(async (conn, driver, engine) => {
+      await testTableCreate(conn, driver, {
+        columns: [
+          {
+            columnName: 'col1',
+            dataType: 'int',
+            notNull: true,
+          },
+          {
+            columnName: 'col2',
+            dataType: 'int',
+            notNull: true,
+          },
+        ],
+        primaryKey: {
+          columns: [{ columnName: 'col1' }],
+        },
+        foreignKeys: [
+          {
+            pureName: 'tested',
+            refTableName: 't0',
+            columns: [{ columnName: 'col2', refColumnName: 'id' }],
+          },
+        ],
+      });
+    })
+  );
+
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Table with unique - %s',
+    testWrapper(async (conn, driver, engine) => {
+      await testTableCreate(conn, driver, {
+        columns: [
+          {
+            columnName: 'col1',
+            dataType: 'int',
+            notNull: true,
+          },
+          {
+            columnName: 'col2',
+            dataType: 'int',
+            notNull: true,
+          },
+        ],
+        primaryKey: {
+          columns: [{ columnName: 'col1' }],
+        },
+        uniques: [
+          {
+            pureName: 'tested',
+            columns: [{ columnName: 'col2' }],
+          },
+        ],
+      });
+    })
+  );
+});

integration-tests/docker-compose.yaml

@@ -0,0 +1,55 @@
+version: '3'
+services:
+  postgres:
+    image: postgres
+    restart: always
+    environment:
+      POSTGRES_PASSWORD: Pwd2020Db
+    ports: 
+      - 15000:5432
+
+  mysql:
+    image: mysql:8.0.18
+    command: --default-authentication-plugin=mysql_native_password
+    restart: always
+    ports: 
+      - 15001:3306
+    environment:
+      - MYSQL_ROOT_PASSWORD=Pwd2020Db
+
+  mssql:
+    image: mcr.microsoft.com/mssql/server
+    restart: always
+    ports: 
+      - 15002:1433
+    environment:
+      - ACCEPT_EULA=Y
+      - SA_PASSWORD=Pwd2020Db
+      - MSSQL_PID=Express
+      
+  cockroachdb:
+    image: cockroachdb/cockroach
+    ports:
+      - 15003:26257
+    command: start-single-node --insecure
+
+#   mongodb:
+#     image: mongo:4.0.12
+#     restart: always
+#     volumes:
+#       - mongo-data:/data/db
+#       - mongo-config:/data/configdb
+#     ports:
+#       - 27017:27017
+
+
+#   cockroachdb-init:
+#     image: cockroachdb/cockroach
+#     # build: cockroach
+#     # entrypoint: /cockroach/init.sh 
+#     entrypoint: ./cockroach sql --insecure --host="cockroachdb" --execute="CREATE DATABASE IF NOT EXISTS test;"
+
+#     depends_on:
+#       - cockroachdb
+#     restart: on-failure
+

integration-tests/engines.js

@@ -0,0 +1,128 @@
+const views = {
+  type: 'views',
+  create1: 'CREATE VIEW obj1 AS SELECT id FROM t1',
+  create2: 'CREATE VIEW obj2 AS SELECT id FROM t2',
+  drop1: 'DROP VIEW obj1',
+  drop2: 'DROP VIEW obj2',
+};
+const matviews = {
+  type: 'matviews',
+  create1: 'CREATE MATERIALIZED VIEW obj1 AS SELECT id FROM t1',
+  create2: 'CREATE MATERIALIZED VIEW obj2 AS SELECT id FROM t2',
+  drop1: 'DROP MATERIALIZED VIEW obj1',
+  drop2: 'DROP MATERIALIZED VIEW obj2',
+};
+
+const engines = [
+  {
+    label: 'MySQL',
+    connection: {
+      engine: 'mysql@dbgate-plugin-mysql',
+      password: 'Pwd2020Db',
+      user: 'root',
+      server: 'mysql',
+      port: 3306,
+    },
+    local: {
+      server: 'localhost',
+      port: 15001,
+    },
+    // skipOnCI: true,
+    objects: [views],
+    dbSnapshotBySeconds: true,
+  },
+  {
+    label: 'PostgreSQL',
+    connection: {
+      engine: 'postgres@dbgate-plugin-postgres',
+      password: 'Pwd2020Db',
+      user: 'postgres',
+      server: 'postgres',
+      port: 5432,
+    },
+    local: {
+      server: 'localhost',
+      port: 15000,
+    },
+    objects: [
+      views,
+      matviews,
+      {
+        type: 'procedures',
+        create1: 'CREATE PROCEDURE obj1() LANGUAGE SQL AS $$  select * from t1 $$',
+        create2: 'CREATE PROCEDURE obj2() LANGUAGE SQL AS $$  select * from t2 $$',
+        drop1: 'DROP PROCEDURE obj1',
+        drop2: 'DROP PROCEDURE obj2',
+      },
+      {
+        type: 'functions',
+        create1:
+          'CREATE FUNCTION obj1() returns int LANGUAGE plpgsql AS $$ declare  res integer; begin select count(*) into res from t1; return res; end; $$',
+        create2:
+          'CREATE FUNCTION obj2() returns int LANGUAGE plpgsql AS $$ declare res integer; begin select count(*) into res from t2; return res; end; $$',
+        drop1: 'DROP FUNCTION obj1',
+        drop2: 'DROP FUNCTION obj2',
+      },
+    ],
+  },
+  {
+    label: 'SQL Server',
+    connection: {
+      engine: 'mssql@dbgate-plugin-mssql',
+      password: 'Pwd2020Db',
+      user: 'sa',
+      server: 'mssql',
+      port: 1433,
+    },
+    local: {
+      server: 'localhost',
+      port: 15002,
+    },
+    objects: [
+      views,
+      {
+        type: 'procedures',
+        create1: 'CREATE PROCEDURE obj1 AS SELECT id FROM t1',
+        create2: 'CREATE PROCEDURE obj2 AS SELECT id FROM t2',
+        drop1: 'DROP PROCEDURE obj1',
+        drop2: 'DROP PROCEDURE obj2',
+      },
+    ],
+  },
+  {
+    label: 'SQLite',
+    generateDbFile: true,
+    connection: {
+      engine: 'sqlite@dbgate-plugin-sqlite',
+    },
+    objects: [views],
+  },
+  {
+    label: 'CockroachDB',
+    connection: {
+      engine: 'cockroach@dbgate-plugin-postgres',
+      user: 'root',
+      server: 'cockroachdb',
+      port: 26257,
+    },
+    local: {
+      server: 'localhost',
+      port: 15003,
+    },
+    skipOnCI: true,
+    objects: [views, matviews],
+  },
+];
+
+const filterLocal = [
+  // filter local testing
+  '-MySQL',
+  '-PostgreSQL',
+  '-SQL Server',
+  'SQLite',
+  '-CockroachDB',
+];
+
+module.exports = process.env.CITEST
+  ? engines.filter(x => !x.skipOnCI)
+  : engines.filter(x => filterLocal.find(y => x.label == y));

integration-tests/package.json

@@ -0,0 +1,28 @@
+{
+  "name": "dbgate-integration-tests",
+  "version": "4.1.1",
+  "homepage": "https://dbgate.org/",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/dbgate/dbgate.git"
+  },
+  "author": "Jan Prochazka",
+  "license": "MIT",
+  "scripts": {
+    "wait:local": "cross-env DEVMODE=1 LOCALTEST=1 node wait.js",
+    "wait:ci": "cross-env DEVMODE=1 CITEST=1 node wait.js",
+
+    "test:local": "cross-env DEVMODE=1 LOCALTEST=1 jest",
+    "test:ci": "cross-env DEVMODE=1 CITEST=1 jest --runInBand --json --outputFile=result.json --testLocationInResults",
+    
+    "run:local": "docker-compose down && docker-compose up -d && yarn wait:local && yarn test:local"
+  },
+  "jest": {
+    "testTimeout": 5000
+  },
+  "devDependencies": {
+    "cross-env": "^7.0.3",
+    "jest": "^27.0.1"
+  },
+  "dependencies": {}
+}

integration-tests/tools.js

@@ -0,0 +1,63 @@
+global.DBGATE_TOOLS = require('dbgate-tools');
+const requireEngineDriver = require('dbgate-api/src/utility/requireEngineDriver');
+const crypto = require('crypto');
+
+function randomDbName() {
+  const generatedKey = crypto.randomBytes(6);
+  const newKey = generatedKey.toString('hex');
+  return `db${newKey}`;
+}
+
+function extractConnection(engine) {
+  const { connection } = engine;
+
+  if (process.env.LOCALTEST && engine.local) {
+    return {
+      ...connection,
+      ...engine.local,
+    };
+  }
+
+  return connection;
+}
+
+async function connect(engine, database) {
+  const connection = extractConnection(engine);
+  const driver = requireEngineDriver(connection);
+
+  if (engine.generateDbFile) {
+    const conn = await driver.connect({
+      ...connection,
+      databaseFile: `dbtemp/${database}`,
+    });
+    return conn;
+  } else {
+    const conn = await driver.connect(connection);
+    await driver.query(conn, `CREATE DATABASE ${database}`);
+    await driver.close(conn);
+
+    const res = await driver.connect({
+      ...connection,
+      database,
+    });
+    return res;
+  }
+}
+
+const testWrapper = body => async (label, ...other) => {
+  const engine = other[other.length - 1];
+  const driver = requireEngineDriver(engine.connection);
+  const conn = await connect(engine, randomDbName());
+  try {
+    await body(conn, driver, ...other);
+  } finally {
+    await driver.close(conn);
+  }
+};
+
+module.exports = {
+  randomDbName,
+  connect,
+  extractConnection,
+  testWrapper,
+};

integration-tests/wait.js

@@ -0,0 +1,29 @@
+const requireEngineDriver = require('dbgate-api/src/utility/requireEngineDriver');
+const engines = require('./engines');
+const { extractConnection } = require('./tools');
+global.DBGATE_TOOLS = require('dbgate-tools');
+
+async function connectEngine(engine) {
+  const connection = extractConnection(engine);
+  const driver = requireEngineDriver(connection);
+  for (;;) {
+    try {
+      const conn = await driver.connect(connection);
+      await driver.getVersion(conn);
+      console.log(`Connect to ${engine.label} - OK`);
+      await driver.close(conn);
+      return;
+    } catch (err) {
+      console.log(`Waiting for ${engine.label}, error: ${err.message}`);
+      await new Promise(resolve => setTimeout(resolve, 2500));
+      continue;
+    }
+  }
+}
+
+async function run() {
+  await new Promise(resolve => setTimeout(resolve, 10000));
+  await Promise.all(engines.map(engine => connectEngine(engine)));
+}
+
+run();

package.json

@@ -1,43 +1,54 @@
 {
   "private": true,
-  "version": "3.9.6",
+  "version": "4.7.4",
   "name": "dbgate-all",
   "workspaces": [
-    "packages/*"
+    "packages/*",
+    "plugins/*",
+    "integration-tests"
   ],
   "scripts": {
     "start:api": "yarn workspace dbgate-api start",
+    "start:app": "cd app && yarn start",
     "start:api:portal": "yarn workspace dbgate-api start:portal",
-    "start:api:covid": "yarn workspace dbgate-api start:covid",
-    "start:web": "yarn workspace dbgate-web start",
+    "start:api:singledb": "yarn workspace dbgate-api start:singledb",
+    "start:web": "yarn workspace dbgate-web dev",
     "start:sqltree": "yarn workspace dbgate-sqltree start",
     "start:tools": "yarn workspace dbgate-tools start",
     "start:datalib": "yarn workspace dbgate-datalib start",
     "start:filterparser": "yarn workspace dbgate-filterparser start",
+    "start:querysplitter": "yarn workspace dbgate-query-splitter start",
     "build:sqltree": "yarn workspace dbgate-sqltree build",
     "build:datalib": "yarn workspace dbgate-datalib build",
     "build:filterparser": "yarn workspace dbgate-filterparser build",
+    "build:querysplitter": "yarn workspace dbgate-query-splitter build",
     "build:tools": "yarn workspace dbgate-tools build",
-    "build:lib": "yarn build:tools && yarn build:sqltree && yarn build:filterparser && yarn build:datalib",
-    "build:app": "cd app && yarn install && yarn build",
+    "build:lib": "yarn build:querysplitter && yarn build:sqltree && yarn build:tools && yarn build:filterparser && yarn build:datalib",
+    "build:app": "yarn plugins:copydist && cd app && yarn install && yarn build",
     "build:api": "yarn workspace dbgate-api build",
-    "build:web:docker": "yarn workspace dbgate-web build:docker",
-    "build:app:local": "cd app && yarn build:local",
+    "build:web:docker": "yarn workspace dbgate-web build",
+    "build:plugins:frontend": "workspaces-run --only=\"dbgate-plugin-*\" -- yarn build:frontend",
+    "build:plugins:frontend:watch": "workspaces-run --parallel --only=\"dbgate-plugin-*\" -- yarn build:frontend:watch",
+    "plugins:copydist": "workspaces-run --only=\"dbgate-plugin-*\" -- yarn copydist",
+    "build:app:local": "yarn plugins:copydist && cd app && yarn build:local",
     "start:app:local": "cd app && yarn start:local",
     "setCurrentVersion": "node setCurrentVersion",
     "generatePadFile": "node generatePadFile",
     "fillNativeModules": "node fillNativeModules",
-    "fillNativeModulesElectron": "node fillNativeModules --eletron",
+    "fillNativeModulesElectron": "node fillNativeModules --electron",
+    "fillPackagedPlugins": "node fillPackagedPlugins",
+    "resetPackagedPlugins": "node resetPackagedPlugins",
     "prettier": "prettier --write packages/api/src && prettier --write packages/datalib/src && prettier --write packages/filterparser/src && prettier --write packages/sqltree/src && prettier --write packages/tools/src && prettier --write packages/types && prettier --write packages/web/src && prettier --write app/src",
-    "copy:docker:build": "copyfiles packages/api/dist/* docker -f && copyfiles packages/web/build/* docker -u 2 && copyfiles \"packages/web/build/**/*\" docker -u 2",
-    "prepare:docker": "yarn build:web:docker && yarn build:api && yarn copy:docker:build",
-    "prepare": "yarn build:lib",
+    "copy:docker:build": "copyfiles packages/api/dist/* docker -f && copyfiles packages/web/public/* docker -u 2 && copyfiles \"packages/web/public/**/*\" docker -u 2 && copyfiles \"plugins/dist/**/*\" docker/plugins -u 2",
+    "install:sqlite:docker": "cd docker && yarn init --yes && yarn add better-sqlite3 && cd ..",
+    "prepare:docker": "yarn plugins:copydist && yarn build:web:docker && yarn build:api && yarn copy:docker:build && yarn install:sqlite:docker",
     "start": "concurrently --kill-others-on-fail \"yarn start:api\" \"yarn start:web\"",
-    "lib": "concurrently --kill-others-on-fail \"yarn start:sqltree\" \"yarn start:filterparser\" \"yarn start:datalib\" \"yarn start:tools\"",
+    "lib": "concurrently --kill-others-on-fail \"yarn start:sqltree\" \"yarn start:filterparser\" \"yarn start:datalib\" \"yarn start:tools\" \"yarn start:querysplitter\" \"yarn build:plugins:frontend:watch\"",
     "ts:api": "yarn workspace dbgate-api ts",
     "ts:web": "yarn workspace dbgate-web ts",
     "ts": "yarn ts:api && yarn ts:web",
-    "postinstall": "patch-package && yarn fillNativeModules"
+    "postinstall": "yarn resetPackagedPlugins && yarn build:lib && patch-package && yarn fillNativeModules && yarn build:plugins:frontend",
+    "dbgate-serve": "node packages/dbgate/bin/dbgate-serve.js"
   },
   "dependencies": {
     "concurrently": "^5.1.0",
@@ -46,6 +57,7 @@
   },
   "devDependencies": {
     "copyfiles": "^2.2.0",
-    "prettier": "^2.2.1"
+    "prettier": "^2.2.1",
+    "workspaces-run": "^1.0.1"
   }
 }

packages/api/.covid-env

@@ -1,12 +0,0 @@
-CONNECTIONS=mysql
-
-LABEL_mysql=MySql
-SERVER_mysql=dbgate.org
-USER_mysql=reader
-PASSWORD_mysql=CovidReader2020
-PORT_mysql=3326
-ENGINE_mysql=mysql@dbgate-plugin-mysql
-SINGLE_CONNECTION=mysql
-SINGLE_DATABASE=covid
-
-PERMISSIONS=files/charts/read

packages/api/.env

@@ -1,23 +1,15 @@
-CONNECTIONS=mysql,postgres
+DEVMODE=1
+# PERMISSIONS=~widgets/app,~widgets/plugins
+# DISABLE_SHELL=1
+# HIDE_APP_EDITOR=1
 
-LABEL_mysql=MySql localhost
-SERVER_mysql=localhost
-USER_mysql=root
-PASSWORD_mysql=test
-PORT_mysql=3307
-ENGINE_mysql=mysql
 
-LABEL_postgres=Postgres localhost
-SERVER_postgres=localhost
-USER_postgres=postgres
-PASSWORD_postgres=test
-PORT_postgres=5433
-ENGINE_postgres=postgres
+# DEVWEB=1
+# LOGINS=admin,test
 
-TOOLBAR=home
-ICON_home=mdi mdi-home
-TITLE_home=Home
-PAGE_home=home.html
-STARTUP_PAGES=home
+# LOGIN_PASSWORD_admin=admin
+# LOGIN_PERMISSIONS_admin=*
 
-PAGES_DIRECTORY=/home/jena/jenasoft/dbgate-web/pages
+# LOGIN_PASSWORD_test=test
+# LOGIN_PERMISSIONS_test=~*, widgets/database
+# WORKSPACE_DIR=/home/jena/dbgate-data-2

packages/api/env/portal/.env

@@ -0,0 +1,44 @@
+DEVMODE=1
+
+CONNECTIONS=mysql,postgres,mongo,mongo2,mysqlssh,sqlite
+
+LABEL_mysql=MySql localhost
+SERVER_mysql=localhost
+USER_mysql=root
+PASSWORD_mysql=test
+PORT_mysql=3307
+ENGINE_mysql=mysql@dbgate-plugin-mysql
+
+LABEL_postgres=Postgres localhost
+SERVER_postgres=localhost
+USER_postgres=postgres
+PASSWORD_postgres=test
+PORT_postgres=5433
+ENGINE_postgres=postgres@dbgate-plugin-postgres
+
+LABEL_mongo=Mongo URL
+URL_mongo=mongodb://localhost:27017
+ENGINE_mongo=mongo@dbgate-plugin-mongo
+
+LABEL_mongo2=Mongo Server
+SERVER_mongo2=localhost
+ENGINE_mongo2=mongo@dbgate-plugin-mongo
+
+LABEL_mysqlssh=MySql SSH
+SERVER_mysqlssh=localhost
+USER_mysqlssh=root
+PASSWORD_mysqlssh=xxx
+PORT_mysqlssh=3316
+ENGINE_mysqlssh=mysql@dbgate-plugin-mysql
+USE_SSH_mysqlssh=1
+SSH_HOST_mysqlssh=demo.dbgate.org
+SSH_PORT_mysqlssh=22
+SSH_MODE_mysqlssh=userPassword
+SSH_LOGIN_mysqlssh=root
+SSH_PASSWORD_mysqlssh=xxx
+
+LABEL_sqlite=sqlite
+FILE_sqlite=/home/jena/dbgate-data/files/sqlite/feeds.sqlite
+ENGINE_sqlite=sqlite@dbgate-plugin-sqlite
+
+# docker run -p 3000:3000 -e CONNECTIONS=mongo -e URL_mongo=mongodb://localhost:27017 -e ENGINE_mongo=mongo@dbgate-plugin-mongo -e LABEL_mongo=mongo dbgate/dbgate:beta

packages/api/env/singledb/.env

@@ -0,0 +1,17 @@
+DEVMODE=1
+
+CONNECTIONS=mysql
+
+LABEL_mysql=MySql localhost
+SERVER_mysql=localhost
+USER_mysql=root
+PASSWORD_mysql=test
+PORT_mysql=3307
+ENGINE_mysql=mysql@dbgate-plugin-mysql
+DBCONFIG_mysql=[{"name":"Chinook","connectionColor":"cyan"}]
+
+
+SINGLE_CONNECTION=mysql
+SINGLE_DATABASE=Chinook
+
+PERMISSIONS=files/charts/read

packages/api/package.json

@@ -1,13 +1,12 @@
 {
   "name": "dbgate-api",
   "main": "src/index.js",
-  "version": "3.9.5",
+  "version": "4.1.1",
   "homepage": "https://dbgate.org/",
   "repository": {
     "type": "git",
     "url": "https://github.com/dbgate/dbgate.git"
   },
-  "funding": "https://www.paypal.com/paypalme/JanProchazkaCz/30eur",
   "author": "Jan Prochazka",
   "license": "MIT",
   "keywords": [
@@ -19,53 +18,61 @@
   ],
   "dependencies": {
     "async-lock": "^1.2.4",
-    "axios": "^0.19.0",
+    "axios": "^0.21.1",
     "body-parser": "^1.19.0",
     "bufferutil": "^4.0.1",
     "byline": "^5.0.0",
     "compare-versions": "^3.6.0",
     "cors": "^2.8.5",
     "cross-env": "^6.0.3",
-    "dbgate-sqltree": "^3.9.5",
-    "dbgate-tools": "^3.9.5",
+    "dbgate-query-splitter": "^4.1.1",
+    "dbgate-sqltree": "^4.1.1",
+    "dbgate-tools": "^4.1.1",
+    "diff": "^5.0.0",
+    "diff2html": "^3.4.13",
     "eslint": "^6.8.0",
     "express": "^4.17.1",
     "express-basic-auth": "^1.2.0",
     "express-fileupload": "^1.2.0",
-    "find-free-port": "^2.0.0",
-    "fs-extra": "^8.1.0",
+    "fs-extra": "^9.1.0",
+    "fs-reverse": "^0.0.3",
+    "get-port": "^5.1.1",
     "http": "^0.0.0",
+    "is-electron": "^2.2.1",
+    "js-yaml": "^4.1.0",
     "json-stable-stringify": "^1.0.1",
     "line-reader": "^0.4.0",
-    "lodash": "^4.17.15",
+    "lodash": "^4.17.21",
     "ncp": "^2.0.0",
-    "nedb-promises": "^4.0.1",
     "node-cron": "^2.0.3",
     "node-ssh-forward": "^0.7.2",
     "portfinder": "^1.0.28",
     "simple-encryptor": "^4.0.0",
-    "socket.io": "^2.3.0",
     "tar": "^6.0.5",
     "uuid": "^3.4.0"
   },
   "scripts": {
-    "start": "nodemon src/index.js",
-    "start:portal": "env-cmd nodemon src/index.js",
-    "start:covid": "env-cmd -f .covid-env nodemon src/index.js",
+    "start": "env-cmd node src/index.js",
+    "start:portal": "env-cmd -f env/portal/.env node src/index.js",
+    "start:singledb": "env-cmd -f env/singledb/.env node src/index.js",
+    "start:filedb": "env-cmd node src/index.js /home/jena/test/chinook/Chinook.db",
+    "start:singleconn": "env-cmd node src/index.js --server localhost --user root --port 3307 --engine mysql@dbgate-plugin-mysql --password test",
     "ts": "tsc",
     "build": "webpack"
   },
   "devDependencies": {
+    "@types/fs-extra": "^9.0.11",
     "@types/lodash": "^4.14.149",
-    "dbgate-types": "^3.9.5",
+    "dbgate-types": "^4.1.1",
     "env-cmd": "^10.1.0",
     "node-loader": "^1.0.2",
     "nodemon": "^2.0.2",
-    "typescript": "^3.7.4",
+    "typescript": "^4.4.3",
     "webpack": "^4.42.0",
     "webpack-cli": "^3.3.11"
   },
   "optionalDependencies": {
-    "msnodesqlv8": "^2.0.10"
+    "better-sqlite3": "7.4.5",
+    "msnodesqlv8": "^2.4.4"
   }
 }

packages/api/src/controllers/apps.js

@@ -0,0 +1,280 @@
+const fs = require('fs-extra');
+const _ = require('lodash');
+const path = require('path');
+const { appdir } = require('../utility/directories');
+const socket = require('../utility/socket');
+const connections = require('./connections');
+
+module.exports = {
+  folders_meta: true,
+  async folders() {
+    const folders = await fs.readdir(appdir());
+    return [
+      ...folders.map(name => ({
+        name,
+      })),
+    ];
+  },
+
+  createFolder_meta: true,
+  async createFolder({ folder }) {
+    const name = await this.getNewAppFolder({ name: folder });
+    await fs.mkdir(path.join(appdir(), name));
+    socket.emitChanged('app-folders-changed');
+    this.emitChangedDbApp(folder);
+    return name;
+  },
+
+  files_meta: true,
+  async files({ folder }) {
+    if (!folder) return [];
+    const dir = path.join(appdir(), folder);
+    if (!(await fs.exists(dir))) return [];
+    const files = await fs.readdir(dir);
+
+    function fileType(ext, type) {
+      return files
+        .filter(name => name.endsWith(ext))
+        .map(name => ({
+          name: name.slice(0, -ext.length),
+          label: path.parse(name.slice(0, -ext.length)).base,
+          type,
+        }));
+    }
+
+    return [
+      ...fileType('.command.sql', 'command.sql'),
+      ...fileType('.query.sql', 'query.sql'),
+      ...fileType('.config.json', 'config.json'),
+    ];
+  },
+
+  async emitChangedDbApp(folder) {
+    const used = await this.getUsedAppFolders();
+    if (used.includes(folder)) {
+      socket.emitChanged('used-apps-changed');
+    }
+  },
+
+  refreshFiles_meta: true,
+  async refreshFiles({ folder }) {
+    socket.emitChanged(`app-files-changed-${folder}`);
+  },
+
+  refreshFolders_meta: true,
+  async refreshFolders() {
+    socket.emitChanged(`app-folders-changed`);
+  },
+
+  deleteFile_meta: true,
+  async deleteFile({ folder, file, fileType }) {
+    await fs.unlink(path.join(appdir(), folder, `${file}.${fileType}`));
+    socket.emitChanged(`app-files-changed-${folder}`);
+    this.emitChangedDbApp(folder);
+  },
+
+  renameFile_meta: true,
+  async renameFile({ folder, file, newFile, fileType }) {
+    await fs.rename(
+      path.join(path.join(appdir(), folder), `${file}.${fileType}`),
+      path.join(path.join(appdir(), folder), `${newFile}.${fileType}`)
+    );
+    socket.emitChanged(`app-files-changed-${folder}`);
+    this.emitChangedDbApp(folder);
+  },
+
+  renameFolder_meta: true,
+  async renameFolder({ folder, newFolder }) {
+    const uniqueName = await this.getNewAppFolder({ name: newFolder });
+    await fs.rename(path.join(appdir(), folder), path.join(appdir(), uniqueName));
+    socket.emitChanged(`app-folders-changed`);
+  },
+
+  deleteFolder_meta: true,
+  async deleteFolder({ folder }) {
+    if (!folder) throw new Error('Missing folder parameter');
+    await fs.rmdir(path.join(appdir(), folder), { recursive: true });
+    socket.emitChanged(`app-folders-changed`);
+    socket.emitChanged(`app-files-changed-${folder}`);
+    socket.emitChanged('used-apps-changed');
+  },
+
+  async getNewAppFolder({ name }) {
+    if (!(await fs.exists(path.join(appdir(), name)))) return name;
+    let index = 2;
+    while (await fs.exists(path.join(appdir(), `${name}${index}`))) {
+      index += 1;
+    }
+    return `${name}${index}`;
+  },
+
+  getUsedAppFolders_meta: true,
+  async getUsedAppFolders() {
+    const list = await connections.list();
+    const apps = [];
+
+    for (const connection of list) {
+      for (const db of connection.databases || []) {
+        for (const key of _.keys(db || {})) {
+          if (key.startsWith('useApp:') && db[key]) {
+            apps.push(key.substring('useApp:'.length));
+          }
+        }
+      }
+    }
+
+    return _.intersection(_.uniq(apps), await fs.readdir(appdir()));
+  },
+
+  getUsedApps_meta: true,
+  async getUsedApps() {
+    const apps = await this.getUsedAppFolders();
+    const res = [];
+
+    for (const folder of apps) {
+      res.push(await this.loadApp({ folder }));
+    }
+    return res;
+  },
+
+  // getAppsForDb_meta: true,
+  // async getAppsForDb({ conid, database }) {
+  //   const connection = await connections.get({ conid });
+  //   if (!connection) return [];
+  //   const db = (connection.databases || []).find(x => x.name == database);
+  //   const apps = [];
+  //   const res = [];
+  //   if (db) {
+  //     for (const key of _.keys(db || {})) {
+  //       if (key.startsWith('useApp:') && db[key]) {
+  //         apps.push(key.substring('useApp:'.length));
+  //       }
+  //     }
+  //   }
+  //   for (const folder of apps) {
+  //     res.push(await this.loadApp({ folder }));
+  //   }
+  //   return res;
+  // },
+
+  loadApp_meta: true,
+  async loadApp({ folder }) {
+    const res = {
+      queries: [],
+      commands: [],
+      name: folder,
+    };
+    const dir = path.join(appdir(), folder);
+    if (await fs.exists(dir)) {
+      const files = await fs.readdir(dir);
+
+      async function processType(ext, field) {
+        for (const file of files) {
+          if (file.endsWith(ext)) {
+            res[field].push({
+              name: file.slice(0, -ext.length),
+              sql: await fs.readFile(path.join(dir, file), { encoding: 'utf-8' }),
+            });
+          }
+        }
+      }
+
+      await processType('.command.sql', 'commands');
+      await processType('.query.sql', 'queries');
+    }
+
+    try {
+      res.virtualReferences = JSON.parse(
+        await fs.readFile(path.join(dir, 'virtual-references.config.json'), { encoding: 'utf-8' })
+      );
+    } catch (err) {
+      res.virtualReferences = [];
+    }
+    try {
+      res.dictionaryDescriptions = JSON.parse(
+        await fs.readFile(path.join(dir, 'dictionary-descriptions.config.json'), { encoding: 'utf-8' })
+      );
+    } catch (err) {
+      res.dictionaryDescriptions = [];
+    }
+
+    return res;
+  },
+
+  async saveConfigFile(appFolder, filename, filterFunc, newItem) {
+    const file = path.join(appdir(), appFolder, filename);
+
+    let json;
+    try {
+      json = JSON.parse(await fs.readFile(file, { encoding: 'utf-8' }));
+    } catch (err) {
+      json = [];
+    }
+
+    if (filterFunc) {
+      json = json.filter(filterFunc);
+    }
+
+    json = [...json, newItem];
+
+    await fs.writeFile(file, JSON.stringify(json, undefined, 2));
+
+    socket.emitChanged(`app-files-changed-${appFolder}`);
+    socket.emitChanged('used-apps-changed');
+  },
+
+  saveVirtualReference_meta: true,
+  async saveVirtualReference({ appFolder, schemaName, pureName, refSchemaName, refTableName, columns }) {
+    await this.saveConfigFile(
+      appFolder,
+      'virtual-references.config.json',
+      columns.length == 1
+        ? x =>
+            !(
+              x.schemaName == schemaName &&
+              x.pureName == pureName &&
+              x.columns.length == 1 &&
+              x.columns[0].columnName == columns[0].columnName
+            )
+        : null,
+      {
+        schemaName,
+        pureName,
+        refSchemaName,
+        refTableName,
+        columns,
+      }
+    );
+    return true;
+  },
+
+  saveDictionaryDescription_meta: true,
+  async saveDictionaryDescription({ appFolder, pureName, schemaName, expression, columns, delimiter }) {
+    await this.saveConfigFile(
+      appFolder,
+      'dictionary-descriptions.config.json',
+      x => !(x.schemaName == schemaName && x.pureName == pureName),
+      {
+        schemaName,
+        pureName,
+        expression,
+        columns,
+        delimiter,
+      }
+    );
+
+    return true;
+  },
+
+  createConfigFile_meta: true,
+  async createConfigFile({ appFolder, fileName, content }) {
+    const file = path.join(appdir(), appFolder, fileName);
+    if (!(await fs.exists(file))) {
+      await fs.writeFile(file, JSON.stringify(content, undefined, 2));
+      socket.emitChanged(`app-files-changed-${appFolder}`);
+      socket.emitChanged('used-apps-changed');
+      return true;
+    }
+    return false;
+  },
+};

packages/api/src/controllers/archive.js

@@ -3,13 +3,14 @@ const stream = require('stream');
 const readline = require('readline');
 const path = require('path');
 const { formatWithOptions } = require('util');
-const { archivedir } = require('../utility/directories');
+const { archivedir, clearArchiveLinksCache, resolveArchiveFolder } = require('../utility/directories');
 const socket = require('../utility/socket');
 const JsonLinesDatastore = require('../utility/JsonLinesDatastore');
 const { saveFreeTableData } = require('../utility/freeTableStorage');
+const loadFilesRecursive = require('../utility/loadFilesRecursive');
 
 module.exports = {
-  folders_meta: 'get',
+  folders_meta: true,
   async folders() {
     const folders = await fs.readdir(archivedir());
     return [
@@ -26,59 +27,103 @@ module.exports = {
     ];
   },
 
-  createFolder_meta: 'post',
+  createFolder_meta: true,
   async createFolder({ folder }) {
     await fs.mkdir(path.join(archivedir(), folder));
     socket.emitChanged('archive-folders-changed');
     return true;
   },
 
-  files_meta: 'get',
-  async files({ folder }) {
-    const dir = path.join(archivedir(), folder);
-    if (!(await fs.exists(dir))) return [];
-    const files = await fs.readdir(dir);
-    return files
-      .filter(name => name.endsWith('.jsonl'))
-      .map(name => ({
-        name: name.slice(0, -'.jsonl'.length),
-        type: 'jsonl',
-      }));
+  createLink_meta: true,
+  async createLink({ linkedFolder }) {
+    const folder = await this.getNewArchiveFolder({ database: path.parse(linkedFolder).name + '.link' });
+    fs.writeFile(path.join(archivedir(), folder), linkedFolder);
+    clearArchiveLinksCache();
+    socket.emitChanged('archive-folders-changed');
+    return folder;
   },
 
-  refreshFiles_meta: 'post',
+  files_meta: true,
+  async files({ folder }) {
+    const dir = resolveArchiveFolder(folder);
+    if (!(await fs.exists(dir))) return [];
+    const files = await loadFilesRecursive(dir); // fs.readdir(dir);
+
+    function fileType(ext, type) {
+      return files
+        .filter(name => name.endsWith(ext))
+        .map(name => ({
+          name: name.slice(0, -ext.length),
+          label: path.parse(name.slice(0, -ext.length)).base,
+          type,
+        }));
+    }
+
+    return [
+      ...fileType('.jsonl', 'jsonl'),
+      ...fileType('.table.yaml', 'table.yaml'),
+      ...fileType('.view.sql', 'view.sql'),
+      ...fileType('.proc.sql', 'proc.sql'),
+      ...fileType('.func.sql', 'func.sql'),
+      ...fileType('.trigger.sql', 'trigger.sql'),
+      ...fileType('.matview.sql', 'matview.sql'),
+    ];
+  },
+
+  refreshFiles_meta: true,
   async refreshFiles({ folder }) {
     socket.emitChanged(`archive-files-changed-${folder}`);
   },
 
-  refreshFolders_meta: 'post',
+  refreshFolders_meta: true,
   async refreshFolders() {
     socket.emitChanged(`archive-folders-changed`);
   },
 
-  deleteFile_meta: 'post',
-  async deleteFile({ folder, file }) {
-    await fs.unlink(path.join(archivedir(), folder, `${file}.jsonl`));
+  deleteFile_meta: true,
+  async deleteFile({ folder, file, fileType }) {
+    await fs.unlink(path.join(resolveArchiveFolder(folder), `${file}.${fileType}`));
     socket.emitChanged(`archive-files-changed-${folder}`);
   },
 
-  deleteFolder_meta: 'post',
-  async deleteFolder({ folder }) {
-    if (!folder) throw new Error('Missing folder parameter');
-    await fs.rmdir(path.join(archivedir(), folder), { recursive: true });
+  renameFile_meta: true,
+  async renameFile({ folder, file, newFile, fileType }) {
+    await fs.rename(
+      path.join(resolveArchiveFolder(folder), `${file}.${fileType}`),
+      path.join(resolveArchiveFolder(folder), `${newFile}.${fileType}`)
+    );
+    socket.emitChanged(`archive-files-changed-${folder}`);
+  },
+
+  renameFolder_meta: true,
+  async renameFolder({ folder, newFolder }) {
+    const uniqueName = await this.getNewArchiveFolder({ database: newFolder });
+    await fs.rename(path.join(archivedir(), folder), path.join(archivedir(), uniqueName));
     socket.emitChanged(`archive-folders-changed`);
   },
 
-  saveFreeTable_meta: 'post',
+  deleteFolder_meta: true,
+  async deleteFolder({ folder }) {
+    if (!folder) throw new Error('Missing folder parameter');
+    if (folder.endsWith('.link')) {
+      await fs.unlink(path.join(archivedir(), folder));
+    } else {
+      await fs.rmdir(path.join(archivedir(), folder), { recursive: true });
+    }
+    socket.emitChanged(`archive-folders-changed`);
+  },
+
+  saveFreeTable_meta: true,
   async saveFreeTable({ folder, file, data }) {
-    saveFreeTableData(path.join(archivedir(), folder, `${file}.jsonl`), data);
+    await saveFreeTableData(path.join(resolveArchiveFolder(folder), `${file}.jsonl`), data);
+    socket.emitChanged(`archive-files-changed-${folder}`);
     return true;
   },
 
-  loadFreeTable_meta: 'post',
+  loadFreeTable_meta: true,
   async loadFreeTable({ folder, file }) {
     return new Promise((resolve, reject) => {
-      const fileStream = fs.createReadStream(path.join(archivedir(), folder, `${file}.jsonl`));
+      const fileStream = fs.createReadStream(path.join(resolveArchiveFolder(folder), `${file}.jsonl`));
       const liner = readline.createInterface({
         input: fileStream,
       });
@@ -95,4 +140,23 @@ module.exports = {
       });
     });
   },
+
+  saveText_meta: true,
+  async saveText({ folder, file, text }) {
+    await fs.writeFile(path.join(resolveArchiveFolder(folder), `${file}.jsonl`), text);
+    socket.emitChanged(`archive-files-changed-${folder}`);
+    return true;
+  },
+
+  async getNewArchiveFolder({ database }) {
+    const isLink = database.endsWith(database);
+    const name = isLink ? database.slice(0, -5) : database;
+    const suffix = isLink ? '.link' : '';
+    if (!(await fs.exists(path.join(archivedir(), database)))) return database;
+    let index = 2;
+    while (await fs.exists(path.join(archivedir(), `${name}${index}${suffix}`))) {
+      index += 1;
+    }
+    return `${name}${index}${suffix}`;
+  },
 };

packages/api/src/controllers/config.js

@@ -1,41 +1,107 @@
+const fs = require('fs-extra');
+const os = require('os');
+const path = require('path');
+const axios = require('axios');
+const { datadir } = require('../utility/directories');
+const { hasPermission, getLogins } = require('../utility/hasPermission');
+const socket = require('../utility/socket');
+const _ = require('lodash');
+const AsyncLock = require('async-lock');
+
 const currentVersion = require('../currentVersion');
 const platformInfo = require('../utility/platformInfo');
+const connections = require('../controllers/connections');
+
+const lock = new AsyncLock();
 
 module.exports = {
-  get_meta: 'get',
-  async get() {
-    // const toolbarButtons = process.env.TOOLBAR;
-    // const toolbar = toolbarButtons
-    //   ? toolbarButtons.split(',').map((name) => ({
-    //       name,
-    //       icon: process.env[`ICON_${name}`],
-    //       title: process.env[`TITLE_${name}`],
-    //       page: process.env[`PAGE_${name}`],
-    //     }))
-    //   : null;
-    // const startupPages = process.env.STARTUP_PAGES ? process.env.STARTUP_PAGES.split(',') : [];
-    const permissions = process.env.PERMISSIONS ? process.env.PERMISSIONS.split(',') : null;
-    const singleDatabase =
-      process.env.SINGLE_CONNECTION && process.env.SINGLE_DATABASE
-        ? {
-            conid: process.env.SINGLE_CONNECTION,
-            database: process.env.SINGLE_DATABASE,
-          }
-        : null;
+  // settingsValue: {},
+
+  // async _init() {
+  //   try {
+  //     this.settingsValue = JSON.parse(await fs.readFile(path.join(datadir(), 'settings.json'), { encoding: 'utf-8' }));
+  //   } catch (err) {
+  //     this.settingsValue = {};
+  //   }
+  // },
+
+  get_meta: true,
+  async get(_params, req) {
+    const logins = getLogins();
+    const login = logins ? logins.find(x => x.login == (req.auth && req.auth.user)) : null;
+    const permissions = login ? login.permissions : null;
 
     return {
-      runAsPortal: !!process.env.CONNECTIONS,
-      // toolbar,
-      // startupPages,
-      singleDatabase,
+      runAsPortal: !!connections.portalConnections,
+      singleDatabase: connections.singleDatabase,
+      // hideAppEditor: !!process.env.HIDE_APP_EDITOR,
+      allowShellConnection: platformInfo.allowShellConnection,
+      allowShellScripting: platformInfo.allowShellConnection,
       permissions,
+      login,
       ...currentVersion,
     };
   },
 
-  platformInfo_meta: 'get',
+  logout_meta: {
+    method: 'get',
+    raw: true,
+  },
+  logout(req, res) {
+    res.status(401).send('Logged out<br><a href="../..">Back to DbGate</a>');
+  },
+
+  platformInfo_meta: true,
   async platformInfo() {
     return platformInfo;
   },
-  
+
+  getSettings_meta: true,
+  async getSettings() {
+    try {
+      return this.fillMissingSettings(
+        JSON.parse(await fs.readFile(path.join(datadir(), 'settings.json'), { encoding: 'utf-8' }))
+      );
+    } catch (err) {
+      return this.fillMissingSettings({});
+    }
+  },
+
+  fillMissingSettings(value) {
+    const res = {
+      ...value,
+    };
+    if (value['app.useNativeMenu'] !== true && value['app.useNativeMenu'] !== false) {
+      res['app.useNativeMenu'] = os.platform() == 'darwin' ? true : false;
+    }
+    return res;
+  },
+
+  updateSettings_meta: true,
+  async updateSettings(values, req) {
+    if (!hasPermission(`settings/change`, req)) return false;
+
+    const res = await lock.acquire('update', async () => {
+      const currentValue = await this.getSettings();
+      try {
+        const updated = {
+          ...currentValue,
+          ...values,
+        };
+        await fs.writeFile(path.join(datadir(), 'settings.json'), JSON.stringify(updated, undefined, 2));
+        // this.settingsValue = updated;
+        socket.emitChanged(`settings-changed`);
+        return updated;
+      } catch (err) {
+        return false;
+      }
+    });
+    return res;
+  },
+
+  changelog_meta: true,
+  async changelog() {
+    const resp = await axios.default.get('https://raw.githubusercontent.com/dbgate/dbgate/master/CHANGELOG.md');
+    return resp.data;
+  },
 };

packages/api/src/controllers/connections.js

@@ -1,89 +1,280 @@
 const path = require('path');
 const { fork } = require('child_process');
 const _ = require('lodash');
-const nedb = require('nedb-promises');
+const fs = require('fs-extra');
 
-const { datadir } = require('../utility/directories');
+const { datadir, filesdir } = require('../utility/directories');
 const socket = require('../utility/socket');
-const { encryptConnection } = require('../utility/crypting');
+const { encryptConnection, maskConnection } = require('../utility/crypting');
 const { handleProcessCommunication } = require('../utility/processComm');
+const { pickSafeConnectionInfo } = require('../utility/crypting');
+const JsonLinesDatabase = require('../utility/JsonLinesDatabase');
+
+const processArgs = require('../utility/processArgs');
+const { safeJsonParse } = require('dbgate-tools');
+const platformInfo = require('../utility/platformInfo');
+
+function getNamedArgs() {
+  const res = {};
+  for (let i = 0; i < process.argv.length; i++) {
+    const name = process.argv[i];
+    if (name.startsWith('--')) {
+      let value = process.argv[i + 1];
+      if (value && value.startsWith('--')) value = null;
+      res[name.substring(2)] = value == null ? true : value;
+      i++;
+    } else {
+      if (name.endsWith('.db') || name.endsWith('.sqlite') || name.endsWith('.sqlite3')) {
+        res.databaseFile = name;
+        res.engine = 'sqlite@dbgate-plugin-sqlite';
+      }
+    }
+  }
+  return res;
+}
+
+function getDatabaseFileLabel(databaseFile) {
+  if (!databaseFile) return databaseFile;
+  const m = databaseFile.match(/[\/]([^\/]+)$/);
+  if (m) return m[1];
+  return databaseFile;
+}
 
 function getPortalCollections() {
   if (process.env.CONNECTIONS) {
-    return _.compact(process.env.CONNECTIONS.split(',')).map(id => ({
+    const connections = _.compact(process.env.CONNECTIONS.split(',')).map(id => ({
       _id: id,
       engine: process.env[`ENGINE_${id}`],
       server: process.env[`SERVER_${id}`],
       user: process.env[`USER_${id}`],
       password: process.env[`PASSWORD_${id}`],
       port: process.env[`PORT_${id}`],
+      databaseUrl: process.env[`URL_${id}`],
+      useDatabaseUrl: !!process.env[`URL_${id}`],
+      databaseFile: process.env[`FILE_${id}`],
+      defaultDatabase:
+        process.env[`DATABASE_${id}`] ||
+        (process.env[`FILE_${id}`] ? getDatabaseFileLabel(process.env[`FILE_${id}`]) : null),
+      singleDatabase: !!process.env[`DATABASE_${id}`] || !!process.env[`FILE_${id}`],
       displayName: process.env[`LABEL_${id}`],
+      isReadOnly: process.env[`READONLY_${id}`],
+      databases: process.env[`DBCONFIG_${id}`] ? safeJsonParse(process.env[`DBCONFIG_${id}`]) : null,
+
+      // SSH tunnel
+      useSshTunnel: process.env[`USE_SSH_${id}`],
+      sshHost: process.env[`SSH_HOST_${id}`],
+      sshPort: process.env[`SSH_PORT_${id}`],
+      sshMode: process.env[`SSH_MODE_${id}`],
+      sshLogin: process.env[`SSH_LOGIN_${id}`],
+      sshPassword: process.env[`SSH_PASSWORD_${id}`],
+      sshKeyfile: process.env[`SSH_KEY_FILE_${id}`],
+      sshKeyfilePassword: process.env[`SSH_KEY_FILE_PASSWORD_${id}`],
+
+      // SSL
+      useSsl: process.env[`USE_SSL_${id}`],
+      sslCaFile: process.env[`SSL_CA_FILE_${id}`],
+      sslCertFile: process.env[`SSL_CERT_FILE_${id}`],
+      sslCertFilePassword: process.env[`SSL_CERT_FILE_PASSWORD_${id}`],
+      sslKeyFile: process.env[`SSL_KEY_FILE_${id}`],
+      sslRejectUnauthorized: process.env[`SSL_REJECT_UNAUTHORIZED_${id}`],
     }));
+    console.log('Using connections from ENV variables:');
+    console.log(JSON.stringify(connections.map(pickSafeConnectionInfo), undefined, 2));
+    const noengine = connections.filter(x => !x.engine);
+    if (noengine.length > 0) {
+      console.log(
+        'Warning: Invalid CONNECTIONS configutation, missing ENGINE for connection ID:',
+        noengine.map(x => x._id)
+      );
+    }
+    return connections;
   }
+
+  const args = getNamedArgs();
+  if (args.databaseFile) {
+    return [
+      {
+        _id: 'argv',
+        databaseFile: args.databaseFile,
+        singleDatabase: true,
+        defaultDatabase: getDatabaseFileLabel(args.databaseFile),
+        engine: args.engine,
+      },
+    ];
+  }
+  if (args.databaseUrl) {
+    return [
+      {
+        _id: 'argv',
+        useDatabaseUrl: true,
+        ...args,
+      },
+    ];
+  }
+  if (args.server) {
+    return [
+      {
+        _id: 'argv',
+        ...args,
+      },
+    ];
+  }
+
   return null;
 }
 const portalConnections = getPortalCollections();
 
+function getSingleDatabase() {
+  if (process.env.SINGLE_CONNECTION && process.env.SINGLE_DATABASE) {
+    // @ts-ignore
+    const connection = portalConnections.find(x => x._id == process.env.SINGLE_CONNECTION);
+    return {
+      connection,
+      name: process.env.SINGLE_DATABASE,
+    };
+  }
+  // @ts-ignore
+  const arg0 = (portalConnections || []).find(x => x._id == 'argv');
+  if (arg0) {
+    // @ts-ignore
+    if (arg0.singleDatabase) {
+      return {
+        connection: arg0,
+        // @ts-ignore
+        name: arg0.defaultDatabase,
+      };
+    }
+  }
+  return null;
+}
+
+const singleDatabase = getSingleDatabase();
+
 module.exports = {
   datastore: null,
   opened: [],
+  singleDatabase,
+  portalConnections,
 
   async _init() {
     const dir = datadir();
     if (!portalConnections) {
       // @ts-ignore
-      this.datastore = nedb.create(path.join(dir, 'connections.jsonl'));
+      this.datastore = new JsonLinesDatabase(path.join(dir, 'connections.jsonl'));
     }
   },
 
-  list_meta: 'get',
+  list_meta: true,
   async list() {
-    return portalConnections || this.datastore.find();
+    return portalConnections && !platformInfo.allowShellConnection
+      ? portalConnections.map(maskConnection)
+      : this.datastore.find();
   },
 
-  test_meta: {
-    method: 'post',
-    raw: true,
-  },
-  test(req, res) {
-    const subprocess = fork(process.argv[1], ['connectProcess', ...process.argv.slice(3)]);
-    subprocess.on('message', resp => {
-      if (handleProcessCommunication(resp, subprocess)) return;
-      // @ts-ignore
-      const { msgtype } = resp;
-      if (msgtype == 'connected' || msgtype == 'error') {
-        res.json(resp);
-      }
+  test_meta: true,
+  test(connection) {
+    const subprocess = fork(global['API_PACKAGE'] || process.argv[1], [
+      '--is-forked-api',
+      '--start-process',
+      'connectProcess',
+      ...processArgs.getPassArgs(),
+      // ...process.argv.slice(3),
+    ]);
+    subprocess.send(connection);
+    return new Promise(resolve => {
+      subprocess.on('message', resp => {
+        if (handleProcessCommunication(resp, subprocess)) return;
+        // @ts-ignore
+        const { msgtype } = resp;
+        if (msgtype == 'connected' || msgtype == 'error') {
+          resolve(resp);
+        }
+      });
     });
-    subprocess.send(req.body);
   },
 
-  save_meta: 'post',
+  save_meta: true,
   async save(connection) {
     if (portalConnections) return;
     let res;
     const encrypted = encryptConnection(connection);
     if (connection._id) {
-      res = await this.datastore.update(_.pick(connection, '_id'), encrypted);
+      res = await this.datastore.update(encrypted);
     } else {
       res = await this.datastore.insert(encrypted);
     }
     socket.emitChanged('connection-list-changed');
+    socket.emitChanged('used-apps-changed');
+    if (this._closeAll) {
+      this._closeAll(connection._id);
+    }
+    // for (const db of connection.databases || []) {
+    //   socket.emitChanged(`db-apps-changed-${connection._id}-${db.name}`);
+    // }
     return res;
   },
 
-  delete_meta: 'post',
-  async delete(connection) {
+  update_meta: true,
+  async update({ _id, values }) {
     if (portalConnections) return;
-    const res = await this.datastore.remove(_.pick(connection, '_id'));
+    const res = await this.datastore.patch(_id, values);
     socket.emitChanged('connection-list-changed');
     return res;
   },
 
-  get_meta: 'get',
+  updateDatabase_meta: true,
+  async updateDatabase({ conid, database, values }) {
+    if (portalConnections) return;
+    const conn = await this.datastore.get(conid);
+    let databases = (conn && conn.databases) || [];
+    if (databases.find(x => x.name == database)) {
+      databases = databases.map(x => (x.name == database ? { ...x, ...values } : x));
+    } else {
+      databases = [...databases, { name: database, ...values }];
+    }
+    const res = await this.datastore.patch(conid, { databases });
+    socket.emitChanged('connection-list-changed');
+    socket.emitChanged('used-apps-changed');
+    // socket.emitChanged(`db-apps-changed-${conid}-${database}`);
+    return res;
+  },
+
+  delete_meta: true,
+  async delete(connection) {
+    if (portalConnections) return;
+    const res = await this.datastore.remove(connection._id);
+    socket.emitChanged('connection-list-changed');
+    return res;
+  },
+
+  async getCore({ conid, mask = false }) {
+    if (!conid) return null;
+    if (portalConnections) {
+      const res = portalConnections.find(x => x._id == conid) || null;
+      return mask && !platformInfo.allowShellConnection ? maskConnection(res) : res;
+    }
+    const res = await this.datastore.get(conid);
+    return res || null;
+  },
+
+  get_meta: true,
   async get({ conid }) {
-    if (portalConnections) return portalConnections.find(x => x._id == conid);
-    const res = await this.datastore.find({ _id: conid });
-    return res[0];
+    return this.getCore({ conid, mask: true });
+  },
+
+  newSqliteDatabase_meta: true,
+  async newSqliteDatabase({ file }) {
+    const sqliteDir = path.join(filesdir(), 'sqlite');
+    if (!(await fs.exists(sqliteDir))) {
+      await fs.mkdir(sqliteDir);
+    }
+    const databaseFile = path.join(sqliteDir, `${file}.sqlite`);
+    const res = await this.save({
+      engine: 'sqlite@dbgate-plugin-sqlite',
+      databaseFile,
+      singleDatabase: true,
+      defaultDatabase: `${file}.sqlite`,
+    });
+    return res;
   },
 };

packages/api/src/controllers/databaseConnections.js

@@ -1,9 +1,31 @@
 const uuidv1 = require('uuid/v1');
 const connections = require('./connections');
+const archive = require('./archive');
 const socket = require('../utility/socket');
 const { fork } = require('child_process');
-const { DatabaseAnalyser } = require('dbgate-tools');
+const {
+  DatabaseAnalyser,
+  computeDbDiffRows,
+  getCreateObjectScript,
+  getAlterDatabaseScript,
+  generateDbPairingId,
+  matchPairedObjects,
+  extendDatabaseInfo,
+  modelCompareDbDiffOptions,
+} = require('dbgate-tools');
+const { html, parse } = require('diff2html');
 const { handleProcessCommunication } = require('../utility/processComm');
+const config = require('./config');
+const fs = require('fs-extra');
+const exportDbModel = require('../utility/exportDbModel');
+const { archivedir, resolveArchiveFolder, uploadsdir } = require('../utility/directories');
+const path = require('path');
+const importDbModel = require('../utility/importDbModel');
+const requireEngineDriver = require('../utility/requireEngineDriver');
+const generateDeploySql = require('../shell/generateDeploySql');
+const { createTwoFilesPatch } = require('diff');
+const diff2htmlPage = require('../utility/diff2htmlPage');
+const processArgs = require('../utility/processArgs');
 
 module.exports = {
   /** @type {import('dbgate-types').OpenedDatabaseConnection[]} */
@@ -11,12 +33,29 @@ module.exports = {
   closed: {},
   requests: {},
 
+  async _init() {
+    connections._closeAll = conid => this.closeAll(conid);
+  },
+
   handle_structure(conid, database, { structure }) {
     const existing = this.opened.find(x => x.conid == conid && x.database == database);
     if (!existing) return;
     existing.structure = structure;
     socket.emitChanged(`database-structure-changed-${conid}-${database}`);
   },
+  handle_structureTime(conid, database, { analysedTime }) {
+    const existing = this.opened.find(x => x.conid == conid && x.database == database);
+    if (!existing) return;
+    existing.analysedTime = analysedTime;
+    socket.emitChanged(`database-status-changed-${conid}-${database}`);
+  },
+  handle_version(conid, database, { version }) {
+    const existing = this.opened.find(x => x.conid == conid && x.database == database);
+    if (!existing) return;
+    existing.serverVersion = version;
+    socket.emitChanged(`database-server-version-changed-${conid}-${database}`);
+  },
+
   handle_error(conid, database, props) {
     const { error } = props;
     console.log(`Error in database connection ${conid}, database ${database}: ${error}`);
@@ -27,9 +66,10 @@ module.exports = {
     delete this.requests[msgid];
   },
   handle_status(conid, database, { status }) {
+    // console.log('HANDLE SET STATUS', status);
     const existing = this.opened.find(x => x.conid == conid && x.database == database);
     if (!existing) return;
-    if (existing.status == status) return;
+    if (existing.status && status && existing.status.counter > status.counter) return;
     existing.status = status;
     socket.emitChanged(`database-status-changed-${conid}-${database}`);
   },
@@ -39,14 +79,21 @@ module.exports = {
   async ensureOpened(conid, database) {
     const existing = this.opened.find(x => x.conid == conid && x.database == database);
     if (existing) return existing;
-    const connection = await connections.get({ conid });
-    const subprocess = fork(process.argv[1], ['databaseConnectionProcess', ...process.argv.slice(3)]);
+    const connection = await connections.getCore({ conid });
+    const subprocess = fork(global['API_PACKAGE'] || process.argv[1], [
+      '--is-forked-api',
+      '--start-process',
+      'databaseConnectionProcess',
+      ...processArgs.getPassArgs(),
+      // ...process.argv.slice(3),
+    ]);
     const lastClosed = this.closed[`${conid}/${database}`];
     const newOpened = {
       conid,
       database,
       subprocess,
       structure: lastClosed ? lastClosed.structure : DatabaseAnalyser.createEmptyStructure(),
+      serverVersion: lastClosed ? lastClosed.serverVersion : null,
       connection,
       status: { name: 'pending' },
     };
@@ -67,6 +114,7 @@ module.exports = {
       msgtype: 'connect',
       connection: { ...connection, database },
       structure: lastClosed ? lastClosed.structure : null,
+      globalSettings: await config.getSettings(),
     });
     return newOpened;
   },
@@ -81,7 +129,7 @@ module.exports = {
     return promise;
   },
 
-  queryData_meta: 'post',
+  queryData_meta: true,
   async queryData({ conid, database, sql }) {
     console.log(`Processing query, conid=${conid}, database=${database}, sql=${sql}`);
     const opened = await this.ensureOpened(conid, database);
@@ -92,38 +140,138 @@ module.exports = {
     return res;
   },
 
-  status_meta: 'get',
+  sqlSelect_meta: true,
+  async sqlSelect({ conid, database, select }) {
+    const opened = await this.ensureOpened(conid, database);
+    const res = await this.sendRequest(opened, { msgtype: 'sqlSelect', select });
+    return res;
+  },
+
+  runScript_meta: true,
+  async runScript({ conid, database, sql }) {
+    console.log(`Processing script, conid=${conid}, database=${database}, sql=${sql}`);
+    const opened = await this.ensureOpened(conid, database);
+    const res = await this.sendRequest(opened, { msgtype: 'runScript', sql });
+    return res;
+  },
+
+  collectionData_meta: true,
+  async collectionData({ conid, database, options }) {
+    const opened = await this.ensureOpened(conid, database);
+    const res = await this.sendRequest(opened, { msgtype: 'collectionData', options });
+    return res.result || null;
+  },
+
+  async loadDataCore(msgtype, { conid, database, ...args }) {
+    const opened = await this.ensureOpened(conid, database);
+    const res = await this.sendRequest(opened, { msgtype, ...args });
+    if (res.errorMessage) {
+      console.error(res.errorMessage);
+
+      return {
+        errorMessage: res.errorMessage,
+      };
+    }
+    return res.result || null;
+  },
+
+  loadKeys_meta: true,
+  async loadKeys({ conid, database, root }) {
+    return this.loadDataCore('loadKeys', { conid, database, root });
+  },
+
+  loadKeyInfo_meta: true,
+  async loadKeyInfo({ conid, database, key }) {
+    return this.loadDataCore('loadKeyInfo', { conid, database, key });
+  },
+
+  loadKeyTableRange_meta: true,
+  async loadKeyTableRange({ conid, database, key, cursor, count }) {
+    return this.loadDataCore('loadKeyTableRange', { conid, database, key, cursor, count });
+  },
+
+  loadFieldValues_meta: true,
+  async loadFieldValues({ conid, database, schemaName, pureName, field, search }) {
+    return this.loadDataCore('loadFieldValues', { conid, database, schemaName, pureName, field, search });
+  },
+
+  callMethod_meta: true,
+  async callMethod({ conid, database, method, args }) {
+    return this.loadDataCore('callMethod', { conid, database, method, args });
+
+    // const opened = await this.ensureOpened(conid, database);
+    // const res = await this.sendRequest(opened, { msgtype: 'callMethod', method, args });
+    // if (res.errorMessage) {
+    //   console.error(res.errorMessage);
+    // }
+    // return res.result || null;
+  },
+
+  updateCollection_meta: true,
+  async updateCollection({ conid, database, changeSet }) {
+    const opened = await this.ensureOpened(conid, database);
+    const res = await this.sendRequest(opened, { msgtype: 'updateCollection', changeSet });
+    if (res.errorMessage) {
+      return {
+        errorMessage: res.errorMessage,
+      };
+    }
+    return res.result || null;
+  },
+
+  status_meta: true,
   async status({ conid, database }) {
     const existing = this.opened.find(x => x.conid == conid && x.database == database);
-    if (existing) return existing.status;
+    if (existing) {
+      return {
+        ...existing.status,
+        analysedTime: existing.analysedTime,
+      };
+    }
     const lastClosed = this.closed[`${conid}/${database}`];
-    if (lastClosed) return lastClosed.status;
+    if (lastClosed) {
+      return {
+        ...lastClosed.status,
+        analysedTime: lastClosed.analysedTime,
+      };
+    }
     return {
       name: 'error',
       message: 'Not connected',
     };
   },
 
-  ping_meta: 'post',
+  ping_meta: true,
   async ping({ conid, database }) {
-    const existing = this.opened.find(x => x.conid == conid && x.database == database);
+    let existing = this.opened.find(x => x.conid == conid && x.database == database);
+
     if (existing) {
       existing.subprocess.send({ msgtype: 'ping' });
+    } else {
+      existing = await this.ensureOpened(conid, database);
     }
+
     return {
       status: 'ok',
       connectionStatus: existing ? existing.status : null,
     };
   },
 
-  refresh_meta: 'post',
-  async refresh({ conid, database }) {
-    this.close(conid, database);
+  refresh_meta: true,
+  async refresh({ conid, database, keepOpen }) {
+    if (!keepOpen) this.close(conid, database);
 
     await this.ensureOpened(conid, database);
     return { status: 'ok' };
   },
 
+  syncModel_meta: true,
+  async syncModel({ conid, database }) {
+    const conn = await this.ensureOpened(conid, database);
+    conn.subprocess.send({ msgtype: 'syncModel' });
+    return { status: 'ok' };
+  },
+
   close(conid, database, kill = true) {
     const existing = this.opened.find(x => x.conid == conid && x.database == database);
     if (existing) {
@@ -141,8 +289,25 @@ module.exports = {
     }
   },
 
-  structure_meta: 'get',
+  closeAll(conid, kill = true) {
+    for (const existing of this.opened.filter(x => x.conid == conid)) {
+      this.close(conid, existing.database, kill);
+    }
+  },
+
+  disconnect_meta: true,
+  async disconnect({ conid, database }) {
+    await this.close(conid, database, true);
+    return { status: 'ok' };
+  },
+
+  structure_meta: true,
   async structure({ conid, database }) {
+    if (conid == '__model') {
+      const model = await importDbModel(database);
+      return model;
+    }
+
     const opened = await this.ensureOpened(conid, database);
     return opened.structure;
     // const existing = this.opened.find((x) => x.conid == conid && x.database == database);
@@ -153,11 +318,116 @@ module.exports = {
     // };
   },
 
-  // runCommand_meta: 'post',
+  serverVersion_meta: true,
+  async serverVersion({ conid, database }) {
+    if (!conid) return null;
+    const opened = await this.ensureOpened(conid, database);
+    return opened.serverVersion || null;
+  },
+
+  sqlPreview_meta: true,
+  async sqlPreview({ conid, database, objects, options }) {
+    // wait for structure
+    await this.structure({ conid, database });
+
+    const opened = await this.ensureOpened(conid, database);
+    const res = await this.sendRequest(opened, { msgtype: 'sqlPreview', objects, options });
+    return res;
+  },
+
+  exportModel_meta: true,
+  async exportModel({ conid, database }) {
+    const archiveFolder = await archive.getNewArchiveFolder({ database });
+    await fs.mkdir(path.join(archivedir(), archiveFolder));
+    const model = await this.structure({ conid, database });
+    await exportDbModel(model, path.join(archivedir(), archiveFolder));
+    socket.emitChanged(`archive-folders-changed`);
+    return { archiveFolder };
+  },
+
+  generateDeploySql_meta: true,
+  async generateDeploySql({ conid, database, archiveFolder }) {
+    const opened = await this.ensureOpened(conid, database);
+    const res = await this.sendRequest(opened, {
+      msgtype: 'generateDeploySql',
+      modelFolder: resolveArchiveFolder(archiveFolder),
+    });
+    return res;
+
+    // const connection = await connections.get({ conid });
+    // return generateDeploySql({
+    //   connection,
+    //   analysedStructure: await this.structure({ conid, database }),
+    //   modelFolder: resolveArchiveFolder(archiveFolder),
+    // });
+
+    // const deployedModel = generateDbPairingId(await importDbModel(path.join(archivedir(), archiveFolder)));
+    // const currentModel = generateDbPairingId(await this.structure({ conid, database }));
+    // const currentModelPaired = matchPairedObjects(deployedModel, currentModel);
+    // const connection = await connections.get({ conid });
+    // const driver = requireEngineDriver(connection);
+    // const { sql } = getAlterDatabaseScript(currentModelPaired, deployedModel, {}, deployedModel, driver);
+    // return {
+    //   deployedModel,
+    //   currentModel,
+    //   currentModelPaired,
+    //   sql,
+    // };
+    // return sql;
+  },
+  // runCommand_meta: true,
   // async runCommand({ conid, database, sql }) {
   //   console.log(`Running SQL command , conid=${conid}, database=${database}, sql=${sql}`);
   //   const opened = await this.ensureOpened(conid, database);
   //   const res = await this.sendRequest(opened, { msgtype: 'queryData', sql });
   //   return res;
   // },
+
+  async getUnifiedDiff({ sourceConid, sourceDatabase, targetConid, targetDatabase }) {
+    const dbDiffOptions = sourceConid == '__model' ? modelCompareDbDiffOptions : {};
+
+    const sourceDb = generateDbPairingId(
+      extendDatabaseInfo(await this.structure({ conid: sourceConid, database: sourceDatabase }))
+    );
+    const targetDb = generateDbPairingId(
+      extendDatabaseInfo(await this.structure({ conid: targetConid, database: targetDatabase }))
+    );
+    // const sourceConnection = await connections.getCore({conid:sourceConid})
+    const connection = await connections.getCore({ conid: targetConid });
+    const driver = requireEngineDriver(connection);
+    const targetDbPaired = matchPairedObjects(sourceDb, targetDb, dbDiffOptions);
+    const diffRows = computeDbDiffRows(sourceDb, targetDbPaired, dbDiffOptions, driver);
+
+    // console.log('sourceDb', sourceDb);
+    // console.log('targetDb', targetDb);
+    // console.log('sourceConid, sourceDatabase', sourceConid, sourceDatabase);
+
+    let res = '';
+    for (const row of diffRows) {
+      // console.log('PAIR', row.source && row.source.pureName, row.target && row.target.pureName);
+      const unifiedDiff = createTwoFilesPatch(
+        (row.target && row.target.pureName) || '',
+        (row.source && row.source.pureName) || '',
+        getCreateObjectScript(row.target, driver),
+        getCreateObjectScript(row.source, driver),
+        '',
+        ''
+      );
+      res += unifiedDiff;
+    }
+    return res;
+  },
+
+  generateDbDiffReport_meta: true,
+  async generateDbDiffReport({ filePath, sourceConid, sourceDatabase, targetConid, targetDatabase }) {
+    const unifiedDiff = await this.getUnifiedDiff({ sourceConid, sourceDatabase, targetConid, targetDatabase });
+
+    const diffJson = parse(unifiedDiff);
+    // $: diffHtml = html(diffJson, { outputFormat: 'side-by-side', drawFileList: false });
+    const diffHtml = html(diffJson, { outputFormat: 'side-by-side' });
+
+    await fs.writeFile(filePath, diff2htmlPage(diffHtml));
+
+    return true;
+  },
 };

packages/api/src/controllers/files.js

@@ -1,9 +1,13 @@
+const uuidv1 = require('uuid/v1');
 const fs = require('fs-extra');
 const path = require('path');
-const { filesdir } = require('../utility/directories');
-const hasPermission = require('../utility/hasPermission');
+const { filesdir, archivedir, resolveArchiveFolder, uploadsdir, appdir } = require('../utility/directories');
+const getChartExport = require('../utility/getChartExport');
+const { hasPermission } = require('../utility/hasPermission');
 const socket = require('../utility/socket');
 const scheduler = require('./scheduler');
+const getDiagramExport = require('../utility/getDiagramExport');
+const apps = require('./apps');
 
 function serialize(format, data) {
   if (format == 'text') return data;
@@ -18,21 +22,21 @@ function deserialize(format, text) {
 }
 
 module.exports = {
-  list_meta: 'get',
-  async list({ folder }) {
-    if (!hasPermission(`files/${folder}/read`)) return [];
+  list_meta: true,
+  async list({ folder }, req) {
+    if (!hasPermission(`files/${folder}/read`, req)) return [];
     const dir = path.join(filesdir(), folder);
     if (!(await fs.exists(dir))) return [];
     const files = (await fs.readdir(dir)).map(file => ({ folder, file }));
     return files;
   },
 
-  listAll_meta: 'get',
-  async listAll() {
+  listAll_meta: true,
+  async listAll(_params, req) {
     const folders = await fs.readdir(filesdir());
     const res = [];
     for (const folder of folders) {
-      if (!hasPermission(`files/${folder}/read`)) continue;
+      if (!hasPermission(`files/${folder}/read`, req)) continue;
       const dir = path.join(filesdir(), folder);
       const files = (await fs.readdir(dir)).map(file => ({ folder, file }));
       res.push(...files);
@@ -40,52 +44,92 @@ module.exports = {
     return res;
   },
 
-  delete_meta: 'post',
-  async delete({ folder, file }) {
-    if (!hasPermission(`files/${folder}/write`)) return;
+  delete_meta: true,
+  async delete({ folder, file }, req) {
+    if (!hasPermission(`files/${folder}/write`, req)) return false;
     await fs.unlink(path.join(filesdir(), folder, file));
     socket.emitChanged(`files-changed-${folder}`);
     socket.emitChanged(`all-files-changed`);
+    return true;
   },
 
-  rename_meta: 'post',
-  async rename({ folder, file, newFile }) {
-    if (!hasPermission(`files/${folder}/write`)) return;
+  rename_meta: true,
+  async rename({ folder, file, newFile }, req) {
+    if (!hasPermission(`files/${folder}/write`, req)) return false;
     await fs.rename(path.join(filesdir(), folder, file), path.join(filesdir(), folder, newFile));
     socket.emitChanged(`files-changed-${folder}`);
     socket.emitChanged(`all-files-changed`);
+    return true;
   },
 
-  load_meta: 'post',
-  async load({ folder, file, format }) {
-    if (!hasPermission(`files/${folder}/read`)) return null;
-    const text = await fs.readFile(path.join(filesdir(), folder, file), { encoding: 'utf-8' });
-    return deserialize(format, text);
-  },
-
-  save_meta: 'post',
-  async save({ folder, file, data, format }) {
-    if (!hasPermission(`files/${folder}/write`)) return;
-    const dir = path.join(filesdir(), folder);
-    if (!(await fs.exists(dir))) {
-      await fs.mkdir(dir);
-    }
-    await fs.writeFile(path.join(dir, file), serialize(format, data));
+  copy_meta: true,
+  async copy({ folder, file, newFile }, req) {
+    if (!hasPermission(`files/${folder}/write`, req)) return false;
+    await fs.copyFile(path.join(filesdir(), folder, file), path.join(filesdir(), folder, newFile));
     socket.emitChanged(`files-changed-${folder}`);
     socket.emitChanged(`all-files-changed`);
-    if (folder == 'shell') {
-      scheduler.reload();
+    return true;
+  },
+
+  load_meta: true,
+  async load({ folder, file, format }, req) {
+    if (folder.startsWith('archive:')) {
+      const text = await fs.readFile(path.join(resolveArchiveFolder(folder.substring('archive:'.length)), file), {
+        encoding: 'utf-8',
+      });
+      return deserialize(format, text);
+    } else if (folder.startsWith('app:')) {
+      const text = await fs.readFile(path.join(appdir(), folder.substring('app:'.length), file), {
+        encoding: 'utf-8',
+      });
+      return deserialize(format, text);
+    } else {
+      if (!hasPermission(`files/${folder}/read`, req)) return null;
+      const text = await fs.readFile(path.join(filesdir(), folder, file), { encoding: 'utf-8' });
+      return deserialize(format, text);
     }
   },
 
-  saveAs_meta: 'post',
+  save_meta: true,
+  async save({ folder, file, data, format }, req) {
+    if (folder.startsWith('archive:')) {
+      if (!hasPermission(`archive/write`, req)) return false;
+      const dir = resolveArchiveFolder(folder.substring('archive:'.length));
+      await fs.writeFile(path.join(dir, file), serialize(format, data));
+      socket.emitChanged(`archive-files-changed-${folder.substring('archive:'.length)}`);
+      return true;
+    } else if (folder.startsWith('app:')) {
+      if (!hasPermission(`apps/write`, req)) return false;
+      const app = folder.substring('app:'.length);
+      await fs.writeFile(path.join(appdir(), app, file), serialize(format, data));
+      socket.emitChanged(`app-files-changed-${app}`);
+      socket.emitChanged('used-apps-changed');
+      apps.emitChangedDbApp(folder);
+      return true;
+    } else {
+      if (!hasPermission(`files/${folder}/write`, req)) return false;
+      const dir = path.join(filesdir(), folder);
+      if (!(await fs.exists(dir))) {
+        await fs.mkdir(dir);
+      }
+      await fs.writeFile(path.join(dir, file), serialize(format, data));
+      socket.emitChanged(`files-changed-${folder}`);
+      socket.emitChanged(`all-files-changed`);
+      if (folder == 'shell') {
+        scheduler.reload();
+      }
+      return true;
+    }
+  },
+
+  saveAs_meta: true,
   async saveAs({ filePath, data, format }) {
     await fs.writeFile(filePath, serialize(format, data));
   },
 
-  favorites_meta: 'get',
-  async favorites() {
-    if (!hasPermission(`files/favorites/read`)) return [];
+  favorites_meta: true,
+  async favorites(_params, req) {
+    if (!hasPermission(`files/favorites/read`, req)) return [];
     const dir = path.join(filesdir(), 'favorites');
     if (!(await fs.exists(dir))) return [];
     const files = await fs.readdir(dir);
@@ -101,4 +145,36 @@ module.exports = {
     }
     return res;
   },
+
+  generateUploadsFile_meta: true,
+  async generateUploadsFile({ extension }) {
+    const fileName = `${uuidv1()}.${extension || 'html'}`;
+    return {
+      fileName,
+      filePath: path.join(uploadsdir(), fileName),
+    };
+  },
+
+  exportChart_meta: true,
+  async exportChart({ filePath, title, config, image }) {
+    const fileName = path.parse(filePath).base;
+    const imageFile = fileName.replace('.html', '-preview.png');
+    const html = getChartExport(title, config, imageFile);
+    await fs.writeFile(filePath, html);
+    if (image) {
+      const index = image.indexOf('base64,');
+      if (index > 0) {
+        const data = image.substr(index + 'base64,'.length);
+        const buf = Buffer.from(data, 'base64');
+        await fs.writeFile(filePath.replace('.html', '-preview.png'), buf);
+      }
+    }
+    return true;
+  },
+
+  exportDiagram_meta: true,
+  async exportDiagram({ filePath, html, css, themeType, themeClassName }) {
+    await fs.writeFile(filePath, getDiagramExport(html, css, themeType, themeClassName));
+    return true;
+  },
 };

packages/api/src/controllers/jsldata.js

@@ -1,6 +1,7 @@
 const fs = require('fs');
 const lineReader = require('line-reader');
 const _ = require('lodash');
+const { __ } = require('lodash/fp');
 const DatastoreProxy = require('../utility/DatastoreProxy');
 const { saveFreeTableData } = require('../utility/freeTableStorage');
 const getJslFileName = require('../utility/getJslFileName');
@@ -10,7 +11,10 @@ const socket = require('../utility/socket');
 function readFirstLine(file) {
   return new Promise((resolve, reject) => {
     lineReader.open(file, (err, reader) => {
-      if (err) reject(err);
+      if (err) {
+        reject(err);
+        return;
+      }
       if (reader.hasNextLine()) {
         reader.nextLine((err, line) => {
           if (err) reject(err);
@@ -104,29 +108,48 @@ module.exports = {
     return datastore;
   },
 
-  getInfo_meta: 'get',
+  getInfo_meta: true,
   async getInfo({ jslid }) {
     const file = getJslFileName(jslid);
-    const firstLine = await readFirstLine(file);
-    if (firstLine) return JSON.parse(firstLine);
-    return null;
+    try {
+      const firstLine = await readFirstLine(file);
+      if (firstLine) {
+        const parsed = JSON.parse(firstLine);
+        if (parsed.__isStreamHeader) {
+          return parsed;
+        }
+        return {
+          __isStreamHeader: true,
+          __isDynamicStructure: true,
+        };
+      }
+      return null;
+    } catch (err) {
+      return null;
+    }
   },
 
-  getRows_meta: 'post',
+  getRows_meta: true,
   async getRows({ jslid, offset, limit, filters }) {
     const datastore = await this.ensureDatastore(jslid);
     return datastore.getRows(offset, limit, _.isEmpty(filters) ? null : filters);
   },
 
-  getStats_meta: 'get',
+  getStats_meta: true,
   getStats({ jslid }) {
     const file = `${getJslFileName(jslid)}.stats`;
-    if (fs.existsSync(file)) return JSON.parse(fs.readFileSync(file, 'utf-8'));
+    if (fs.existsSync(file)) {
+      try {
+        return JSON.parse(fs.readFileSync(file, 'utf-8'));
+      } catch (e) {
+        return {};
+      }
+    }
     return {};
   },
 
   async notifyChangedStats(stats) {
-    console.log('SENDING STATS', JSON.stringify(stats));
+    // console.log('SENDING STATS', JSON.stringify(stats));
     const datastore = this.datastores[stats.jslid];
     if (datastore) await datastore.notifyChanged();
     socket.emit(`jsldata-stats-${stats.jslid}`, stats);
@@ -140,9 +163,15 @@ module.exports = {
     // }
   },
 
-  saveFreeTable_meta: 'post',
+  saveFreeTable_meta: true,
   async saveFreeTable({ jslid, data }) {
     saveFreeTableData(getJslFileName(jslid), data);
     return true;
   },
+
+  saveText_meta: true,
+  async saveText({ jslid, text }) {
+    await fs.promises.writeFile(getJslFileName(jslid), text);
+    return true;
+  },
 };

packages/api/src/controllers/metadata.js

@@ -7,17 +7,17 @@ function pickObjectNames(array) {
 }
 
 module.exports = {
-  // tableData_meta: 'get',
+  // tableData_meta: true,
   // async tableData({ conid, database, schemaName, pureName }) {
   //   const opened = await databaseConnections.ensureOpened(conid, database);
   //   const res = await databaseConnections.sendRequest(opened, { msgtype: 'tableData', schemaName, pureName });
   //   return res;
   // },
 
-  listObjects_meta: 'get',
+  listObjects_meta: true,
   async listObjects({ conid, database }) {
     const opened = await databaseConnections.ensureOpened(conid, database);
-    const types = ['tables', 'views', 'procedures', 'functions', 'triggers'];
+    const types = ['tables', 'collections', 'views', 'procedures', 'functions', 'triggers'];
     return types.reduce(
       (res, type) => ({
         ...res,
@@ -27,7 +27,7 @@ module.exports = {
     );
   },
 
-  tableInfo_meta: 'get',
+  tableInfo_meta: true,
   async tableInfo({ conid, database, schemaName, pureName }) {
     const opened = await databaseConnections.ensureOpened(conid, database);
     const table = opened.structure.tables.find(x => x.pureName == pureName && x.schemaName == schemaName);
@@ -38,7 +38,7 @@ module.exports = {
     };
   },
 
-  sqlObjectInfo_meta: 'get',
+  sqlObjectInfo_meta: true,
   async sqlObjectInfo({ objectTypeField, conid, database, schemaName, pureName }) {
     const opened = await databaseConnections.ensureOpened(conid, database);
     const res = opened.structure[objectTypeField].find(x => x.pureName == pureName && x.schemaName == schemaName);

packages/api/src/controllers/plugins.js

@@ -2,50 +2,35 @@ const fs = require('fs-extra');
 const axios = require('axios');
 const path = require('path');
 const { extractPackageName } = require('dbgate-tools');
-const { pluginsdir, datadir } = require('../utility/directories');
+const { pluginsdir, packagedPluginsDir } = require('../utility/directories');
 const socket = require('../utility/socket');
 const compareVersions = require('compare-versions');
 const requirePlugin = require('../shell/requirePlugin');
 const downloadPackage = require('../utility/downloadPackage');
-const hasPermission = require('../utility/hasPermission');
-
-// async function loadPackageInfo(dir) {
-//   const readmeFile = path.join(dir, 'README.md');
-//   const packageFile = path.join(dir, 'package.json');
-
-//   if (!(await fs.exists(packageFile))) {
-//     return null;
-//   }
-
-//   let readme = null;
-//   let manifest = null;
-//   if (await fs.exists(readmeFile)) readme = await fs.readFile(readmeFile, { encoding: 'utf-8' });
-//   if (await fs.exists(packageFile)) manifest = JSON.parse(await fs.readFile(packageFile, { encoding: 'utf-8' }));
-//   return {
-//     readme,
-//     manifest,
-//   };
-// }
-
-const preinstallPluginMinimalVersions = {
-  'dbgate-plugin-mssql': '1.1.0',
-  'dbgate-plugin-mysql': '1.1.0',
-  'dbgate-plugin-postgres': '1.1.0',
-  'dbgate-plugin-csv': '1.0.8',
-  'dbgate-plugin-excel': '1.0.6',
-};
+const { hasPermission } = require('../utility/hasPermission');
+const _ = require('lodash');
+const packagedPluginsContent = require('../packagedPluginsContent');
 
 module.exports = {
-  script_meta: 'get',
+  script_meta: true,
   async script({ packageName }) {
-    const file = path.join(pluginsdir(), packageName, 'dist', 'frontend.js');
+    const packagedContent = packagedPluginsContent();
+
+    if (packagedContent && packagedContent[packageName]) {
+      return packagedContent[packageName].frontend;
+    }
+
+    const file1 = path.join(packagedPluginsDir(), packageName, 'dist', 'frontend.js');
+    const file2 = path.join(pluginsdir(), packageName, 'dist', 'frontend.js');
+    // @ts-ignore
+    const file = (await fs.exists(file1)) ? file1 : file2;
     const data = await fs.readFile(file, {
       encoding: 'utf-8',
     });
     return data;
   },
 
-  search_meta: 'get',
+  search_meta: true,
   async search({ filter }) {
     // DOCS: https://github.com/npm/registry/blob/master/docs/REGISTRY-API.md#get-v1search
     const resp = await axios.default.get(
@@ -55,17 +40,20 @@ module.exports = {
     return (objects || []).map(x => x.package);
   },
 
-  info_meta: 'get',
+  info_meta: true,
   async info({ packageName }) {
     try {
       const infoResp = await axios.default.get(`https://registry.npmjs.org/${packageName}`);
       const { latest } = infoResp.data['dist-tags'];
       const manifest = infoResp.data.versions[latest];
       const { readme } = infoResp.data;
+      // @ts-ignore
+      const isPackaged = await fs.exists(path.join(packagedPluginsDir(), packageName));
 
       return {
         readme,
         manifest,
+        isPackaged,
       };
     } catch (err) {
       return {
@@ -73,68 +61,87 @@ module.exports = {
         error: err.message,
       };
     }
-
-    // const dir = path.join(pluginstmpdir(), packageName);
-    // if (!(await fs.exists(dir))) {
-    //   await downloadPackage(packageName, dir);
-    // }
-    // return await loadPackageInfo(dir);
-    // return await {
-    //   ...loadPackageInfo(dir),
-    //   installed: loadPackageInfo(path.join(pluginsdir(), packageName)),
-    // };
   },
 
-  installed_meta: 'get',
+  installed_meta: true,
   async installed() {
-    const files = await fs.readdir(pluginsdir());
+    const packagedContent = packagedPluginsContent();
+
+    const files1 = packagedContent ? _.keys(packagedContent) : await fs.readdir(packagedPluginsDir());
+    const files2 = await fs.readdir(pluginsdir());
+
     const res = [];
-    for (const packageName of files) {
-      const manifest = await fs.readFile(path.join(pluginsdir(), packageName, 'package.json')).then(x => JSON.parse(x));
-      const readmeFile = path.join(pluginsdir(), packageName, 'README.md');
-      if (await fs.exists(readmeFile)) {
-        manifest.readme = await fs.readFile(readmeFile, { encoding: 'utf-8' });
+    for (const packageName of _.union(files1, files2)) {
+      if (packageName == 'dist') continue;
+      if (!/^dbgate-plugin-.*$/.test(packageName)) continue;
+      try {
+        if (packagedContent && packagedContent[packageName]) {
+          const manifest = {
+            ...packagedContent[packageName].manifest,
+          };
+          manifest.isPackaged = true;
+          manifest.readme = packagedContent[packageName].readme;
+          res.push(manifest);
+        } else {
+          const isPackaged = files1.includes(packageName);
+          const manifest = await fs
+            .readFile(path.join(isPackaged ? packagedPluginsDir() : pluginsdir(), packageName, 'package.json'), {
+              encoding: 'utf-8',
+            })
+            .then(x => JSON.parse(x));
+          if (!manifest.keywords) {
+            continue;
+          }
+          if (!manifest.keywords.includes('dbgateplugin')) {
+            continue;
+          }
+          const readmeFile = path.join(isPackaged ? packagedPluginsDir() : pluginsdir(), packageName, 'README.md');
+          // @ts-ignore
+          if (await fs.exists(readmeFile)) {
+            manifest.readme = await fs.readFile(readmeFile, { encoding: 'utf-8' });
+          }
+          manifest.isPackaged = isPackaged;
+          res.push(manifest);
+        }
+      } catch (err) {
+        console.log(`Skipped plugin ${packageName}, error:`, err.message);
       }
-      res.push(manifest);
     }
     return res;
-    // const res = await Promise.all(
-    //   files.map((packageName) =>
-    //     fs.readFile(path.join(pluginsdir(), packageName, 'package.json')).then((x) => JSON.parse(x))
-    //   )
-    // );
   },
 
-  async saveRemovePlugins() {
-    await fs.writeFile(path.join(datadir(), 'removed-plugins'), this.removedPlugins.join('\n'));
-  },
+  // async saveRemovePlugins() {
+  //   await fs.writeFile(path.join(datadir(), 'removed-plugins'), this.removedPlugins.join('\n'));
+  // },
 
-  install_meta: 'post',
-  async install({ packageName }) {
-    if (!hasPermission(`plugins/install`)) return;
+  install_meta: true,
+  async install({ packageName }, req) {
+    if (!hasPermission(`plugins/install`, req)) return;
     const dir = path.join(pluginsdir(), packageName);
+    // @ts-ignore
     if (!(await fs.exists(dir))) {
       await downloadPackage(packageName, dir);
     }
     socket.emitChanged(`installed-plugins-changed`);
-    this.removedPlugins = this.removedPlugins.filter(x => x != packageName);
-    await this.saveRemovePlugins();
+    // this.removedPlugins = this.removedPlugins.filter(x => x != packageName);
+    // await this.saveRemovePlugins();
   },
 
-  uninstall_meta: 'post',
-  async uninstall({ packageName }) {
-    if (!hasPermission(`plugins/install`)) return;
+  uninstall_meta: true,
+  async uninstall({ packageName }, req) {
+    if (!hasPermission(`plugins/install`, req)) return;
     const dir = path.join(pluginsdir(), packageName);
     await fs.rmdir(dir, { recursive: true });
     socket.emitChanged(`installed-plugins-changed`);
-    this.removedPlugins.push(packageName);
+    // this.removedPlugins.push(packageName);
     await this.saveRemovePlugins();
   },
 
-  upgrade_meta: 'post',
-  async upgrade({ packageName }) {
-    if (!hasPermission(`plugins/install`)) return;
+  upgrade_meta: true,
+  async upgrade({ packageName }, req) {
+    if (!hasPermission(`plugins/install`, req)) return;
     const dir = path.join(pluginsdir(), packageName);
+    // @ts-ignore
     if (await fs.exists(dir)) {
       await fs.rmdir(dir, { recursive: true });
       await downloadPackage(packageName, dir);
@@ -143,60 +150,61 @@ module.exports = {
     socket.emitChanged(`installed-plugins-changed`);
   },
 
-  command_meta: 'post',
+  command_meta: true,
   async command({ packageName, command, args }) {
     const content = requirePlugin(packageName);
     return content.commands[command](args);
   },
 
-  authTypes_meta: 'post',
+  authTypes_meta: true,
   async authTypes({ engine }) {
     const packageName = extractPackageName(engine);
     const content = requirePlugin(packageName);
-    if (!content.driver || content.driver.engine != engine || !content.driver.getAuthTypes) return null;
-    return content.driver.getAuthTypes() || null;
+    const driver = content.drivers.find(x => x.engine == engine);
+    if (!driver || !driver.getAuthTypes) return null;
+    return driver.getAuthTypes() || null;
   },
 
-  async _init() {
-    const installed = await this.installed();
-    try {
-      this.removedPlugins = (await fs.readFile(path.join(datadir(), 'removed-plugins'), { encoding: 'utf-8' })).split(
-        '\n'
-      );
-    } catch (err) {
-      this.removedPlugins = [];
-    }
+  // async _init() {
+  //   const installed = await this.installed();
+  //   try {
+  //     this.removedPlugins = (await fs.readFile(path.join(datadir(), 'removed-plugins'), { encoding: 'utf-8' })).split(
+  //       '\n'
+  //     );
+  //   } catch (err) {
+  //     this.removedPlugins = [];
+  //   }
 
-    for (const packageName of Object.keys(preinstallPluginMinimalVersions)) {
-      const installedVersion = installed.find(x => x.name == packageName);
-      if (installedVersion) {
-        // plugin installed, test, whether upgrade
-        const requiredVersion = preinstallPluginMinimalVersions[packageName];
-        if (compareVersions(installedVersion.version, requiredVersion) < 0) {
-          console.log(
-            `Upgrading preinstalled plugin ${packageName}, found ${installedVersion.version}, required version ${requiredVersion}`
-          );
-          await this.upgrade({ packageName });
-        } else {
-          console.log(
-            `Plugin ${packageName} not upgraded, found ${installedVersion.version}, required version ${requiredVersion}`
-          );
-        }
+  //   for (const packageName of Object.keys(preinstallPluginMinimalVersions)) {
+  //     const installedVersion = installed.find(x => x.name == packageName);
+  //     if (installedVersion) {
+  //       // plugin installed, test, whether upgrade
+  //       const requiredVersion = preinstallPluginMinimalVersions[packageName];
+  //       if (compareVersions(installedVersion.version, requiredVersion) < 0) {
+  //         console.log(
+  //           `Upgrading preinstalled plugin ${packageName}, found ${installedVersion.version}, required version ${requiredVersion}`
+  //         );
+  //         await this.upgrade({ packageName });
+  //       } else {
+  //         console.log(
+  //           `Plugin ${packageName} not upgraded, found ${installedVersion.version}, required version ${requiredVersion}`
+  //         );
+  //       }
 
-        continue;
-      }
+  //       continue;
+  //     }
 
-      if (this.removedPlugins.includes(packageName)) {
-        // plugin was remvoed, don't install again
-        continue;
-      }
+  //     if (this.removedPlugins.includes(packageName)) {
+  //       // plugin was remvoed, don't install again
+  //       continue;
+  //     }
 
-      try {
-        console.log('Preinstalling plugin', packageName);
-        await this.install({ packageName });
-      } catch (err) {
-        console.error('Error preinstalling plugin', packageName, err);
-      }
-    }
-  },
+  //     try {
+  //       console.log('Preinstalling plugin', packageName);
+  //       await this.install({ packageName });
+  //     } catch (err) {
+  //       console.error('Error preinstalling plugin', packageName, err);
+  //     }
+  //   }
+  // },
 };

packages/api/src/controllers/queryHistory.js

@@ -0,0 +1,54 @@
+const fsReverse = require('fs-reverse');
+const fs = require('fs-extra');
+const path = require('path');
+const { datadir } = require('../utility/directories');
+const _ = require('lodash');
+const { filterName } = require('dbgate-tools');
+const socket = require('../utility/socket');
+
+function readCore(reader, skip, limit, filter) {
+  return new Promise((resolve, reject) => {
+    const res = [];
+    let readed = 0;
+    reader.on('data', line => {
+      if (!line && !line.trim()) return;
+      try {
+        const json = JSON.parse(line);
+        if (filterName(filter, json.sql, json.database)) {
+          if (!skip || readed >= skip) {
+            res.push(json);
+          }
+          readed++;
+          if (limit && readed > (skip || 0) + limit) {
+            reader.destroy();
+            resolve(res);
+          }
+        }
+      } catch (err) {
+        reader.destroy();
+        reject(err);
+      }
+    });
+    reader.on('end', () => resolve(res));
+  });
+}
+
+module.exports = {
+  read_meta: true,
+  async read({ skip, limit, filter }) {
+    const fileName = path.join(datadir(), 'query-history.jsonl');
+    // @ts-ignore
+    if (!(await fs.exists(fileName))) return [];
+    const reader = fsReverse(fileName);
+    const res = await readCore(reader, skip, limit, filter);
+    return res;
+  },
+
+  write_meta: true,
+  async write({ data }) {
+    const fileName = path.join(datadir(), 'query-history.jsonl');
+    await fs.appendFile(fileName, JSON.stringify(data) + '\n');
+    socket.emit('query-history-changed');
+    return 'OK';
+  },
+};

packages/api/src/controllers/runners.js

@@ -5,9 +5,11 @@ const uuidv1 = require('uuid/v1');
 const byline = require('byline');
 const socket = require('../utility/socket');
 const { fork } = require('child_process');
-const { rundir, uploadsdir, pluginsdir } = require('../utility/directories');
-const { extractShellApiPlugins, extractShellApiFunctionName } = require('dbgate-tools');
+const { rundir, uploadsdir, pluginsdir, getPluginBackendPath, packagedPluginList } = require('../utility/directories');
+const { extractShellApiPlugins, extractShellApiFunctionName, jsonScriptToJavascript } = require('dbgate-tools');
 const { handleProcessCommunication } = require('../utility/processComm');
+const processArgs = require('../utility/processArgs');
+const platformInfo = require('../utility/platformInfo');
 
 function extractPlugins(script) {
   const requireRegex = /\s*\/\/\s*@require\s+([^\s]+)\s*\n/g;
@@ -15,17 +17,20 @@ function extractPlugins(script) {
   return matches.map(x => x[1]);
 }
 
-const requirePluginsTemplate = plugins =>
+const requirePluginsTemplate = (plugins, isExport) =>
   plugins
     .map(
-      packageName => `const ${_.camelCase(packageName)} = require(process.env.PLUGIN_${_.camelCase(packageName)});\n`
+      packageName =>
+        `const ${_.camelCase(packageName)} = require(${
+          isExport ? `'${packageName}'` : `process.env.PLUGIN_${_.camelCase(packageName)}`
+        });\n`
     )
     .join('') + `dbgateApi.registerPlugins(${plugins.map(x => _.camelCase(x)).join(',')});\n`;
 
-const scriptTemplate = script => `
-const dbgateApi = require(process.env.DBGATE_API);
+const scriptTemplate = (script, isExport) => `
+const dbgateApi = require(${isExport ? `'dbgate-api'` : 'process.env.DBGATE_API'});
 dbgateApi.initializeApiEnvironment();
-${requirePluginsTemplate(extractPlugins(script))}
+${requirePluginsTemplate(extractPlugins(script), isExport)}
 require=null;
 async function run() {
 ${script}
@@ -92,18 +97,25 @@ module.exports = {
     const scriptFile = path.join(uploadsdir(), runid + '.js');
     fs.writeFileSync(`${scriptFile}`, scriptText);
     fs.mkdirSync(directory);
-    const pluginNames = fs.readdirSync(pluginsdir());
+    const pluginNames = _.union(fs.readdirSync(pluginsdir()), packagedPluginList);
     console.log(`RUNNING SCRIPT ${scriptFile}`);
     // const subprocess = fork(scriptFile, ['--checkParent', '--max-old-space-size=8192'], {
-    const subprocess = fork(scriptFile, ['--checkParent', ...process.argv.slice(3)], {
-      cwd: directory,
-      stdio: ['ignore', 'pipe', 'pipe', 'ipc'],
-      env: {
-        ...process.env,
-        DBGATE_API: global['dbgateApiModulePath'] || process.argv[1],
-        ..._.fromPairs(pluginNames.map(name => [`PLUGIN_${_.camelCase(name)}`, path.join(pluginsdir(), name)])),
-      },
-    });
+    const subprocess = fork(
+      scriptFile,
+      [
+        '--checkParent', // ...process.argv.slice(3)
+        ...processArgs.getPassArgs(),
+      ],
+      {
+        cwd: directory,
+        stdio: ['ignore', 'pipe', 'pipe', 'ipc'],
+        env: {
+          ...process.env,
+          DBGATE_API: global['API_PACKAGE'] || process.argv[1],
+          ..._.fromPairs(pluginNames.map(name => [`PLUGIN_${_.camelCase(name)}`, getPluginBackendPath(name)])),
+        },
+      }
+    );
     const pipeDispatcher = severity => data =>
       this.dispatchMessage(runid, { severity, message: data.toString().trim() });
 
@@ -133,16 +145,31 @@ module.exports = {
       if (handleProcessCommunication(message, subprocess)) return;
       this[`handle_${msgtype}`](runid, message);
     });
-    return newOpened;
+    return _.pick(newOpened, ['runid']);
   },
 
-  start_meta: 'post',
+  start_meta: true,
   async start({ script }) {
     const runid = uuidv1();
-    return this.startCore(runid, scriptTemplate(script));
+
+    if (script.type == 'json') {
+      const js = jsonScriptToJavascript(script);
+      return this.startCore(runid, scriptTemplate(js, false));
+    }
+
+    if (!platformInfo.allowShellScripting) {
+      return { errorMessage: 'Shell scripting is not allowed' };
+    }
+
+    return this.startCore(runid, scriptTemplate(script, false));
   },
 
-  cancel_meta: 'post',
+  getNodeScript_meta: true,
+  async getNodeScript({ script }) {
+    return scriptTemplate(script, true);
+  },
+
+  cancel_meta: true,
   async cancel({ runid }) {
     const runner = this.opened.find(x => x.runid == runid);
     if (!runner) {
@@ -152,7 +179,7 @@ module.exports = {
     return { state: 'ok' };
   },
 
-  files_meta: 'get',
+  files_meta: true,
   async files({ runid }) {
     const directory = path.join(rundir(), runid);
     const files = await fs.readdir(directory);
@@ -168,7 +195,7 @@ module.exports = {
     return res;
   },
 
-  loadReader_meta: 'post',
+  loadReader_meta: true,
   async loadReader({ functionName, props }) {
     const promise = new Promise((resolve, reject) => {
       const runid = uuidv1();

packages/api/src/controllers/scheduler.js

@@ -3,7 +3,7 @@ const fs = require('fs-extra');
 const path = require('path');
 const cron = require('node-cron');
 const runners = require('./runners');
-const hasPermission = require('../utility/hasPermission');
+const { hasPermission } = require('../utility/hasPermission');
 
 const scheduleRegex = /\s*\/\/\s*@schedule\s+([^\n]+)\n/;
 
@@ -26,8 +26,8 @@ module.exports = {
     this.tasks.push(task);
   },
 
-  async reload() {
-    if (!hasPermission('files/shell/read')) return;
+  async reload(_params, req) {
+    if (!hasPermission('files/shell/read', req)) return;
     const shellDir = path.join(filesdir(), 'shell');
     await this.unload();
     if (!(await fs.exists(shellDir))) return;

packages/api/src/controllers/serverConnections.js

@@ -5,6 +5,8 @@ const _ = require('lodash');
 const AsyncLock = require('async-lock');
 const { handleProcessCommunication } = require('../utility/processComm');
 const lock = new AsyncLock();
+const config = require('./config');
+const processArgs = require('../utility/processArgs');
 
 module.exports = {
   opened: [],
@@ -17,6 +19,12 @@ module.exports = {
     existing.databases = databases;
     socket.emitChanged(`database-list-changed-${conid}`);
   },
+  handle_version(conid, { version }) {
+    const existing = this.opened.find(x => x.conid == conid);
+    if (!existing) return;
+    existing.version = version;
+    socket.emitChanged(`server-version-changed-${conid}`);
+  },
   handle_status(conid, { status }) {
     const existing = this.opened.find(x => x.conid == conid);
     if (!existing) return;
@@ -29,8 +37,14 @@ module.exports = {
     const res = await lock.acquire(conid, async () => {
       const existing = this.opened.find(x => x.conid == conid);
       if (existing) return existing;
-      const connection = await connections.get({ conid });
-      const subprocess = fork(process.argv[1], ['serverConnectionProcess', ...process.argv.slice(3)]);
+      const connection = await connections.getCore({ conid });
+      const subprocess = fork(global['API_PACKAGE'] || process.argv[1], [
+        '--is-forked-api',
+        '--start-process',
+        'serverConnectionProcess',
+        ...processArgs.getPassArgs(),
+        // ...process.argv.slice(3),
+      ]);
       const newOpened = {
         conid,
         subprocess,
@@ -55,7 +69,7 @@ module.exports = {
         if (newOpened.disconnected) return;
         this.close(conid, false);
       });
-      subprocess.send({ msgtype: 'connect', ...connection });
+      subprocess.send({ msgtype: 'connect', ...connection, globalSettings: await config.getSettings() });
       return newOpened;
     });
     return res;
@@ -75,13 +89,25 @@ module.exports = {
     }
   },
 
-  listDatabases_meta: 'get',
+  disconnect_meta: true,
+  async disconnect({ conid }) {
+    await this.close(conid, true);
+    return { status: 'ok' };
+  },
+
+  listDatabases_meta: true,
   async listDatabases({ conid }) {
     const opened = await this.ensureOpened(conid);
     return opened.databases;
   },
 
-  serverStatus_meta: 'get',
+  version_meta: true,
+  async version({ conid }) {
+    const opened = await this.ensureOpened(conid);
+    return opened.version;
+  },
+
+  serverStatus_meta: true,
   async serverStatus() {
     return {
       ...this.closed,
@@ -89,7 +115,7 @@ module.exports = {
     };
   },
 
-  ping_meta: 'post',
+  ping_meta: true,
   async ping({ connections }) {
     await Promise.all(
       _.uniq(connections).map(async conid => {
@@ -105,15 +131,15 @@ module.exports = {
     return { status: 'ok' };
   },
 
-  refresh_meta: 'post',
-  async refresh({ conid }) {
-    this.close(conid);
+  refresh_meta: true,
+  async refresh({ conid, keepOpen }) {
+    if (!keepOpen) this.close(conid);
 
     await this.ensureOpened(conid);
     return { status: 'ok' };
   },
 
-  createDatabase_meta: 'post',
+  createDatabase_meta: true,
   async createDatabase({ conid, name }) {
     const opened = await this.ensureOpened(conid);
     opened.subprocess.send({ msgtype: 'createDatabase', name });

packages/api/src/controllers/sessions.js

@@ -4,7 +4,10 @@ const connections = require('./connections');
 const socket = require('../utility/socket');
 const { fork } = require('child_process');
 const jsldata = require('./jsldata');
+const path = require('path');
 const { handleProcessCommunication } = require('../utility/processComm');
+const processArgs = require('../utility/processArgs');
+const { appdir } = require('../utility/directories');
 
 module.exports = {
   /** @type {import('dbgate-types').OpenedSession[]} */
@@ -45,9 +48,18 @@ module.exports = {
     this.dispatchMessage(sesid, info);
   },
 
-  handle_done(sesid) {
+  handle_done(sesid, props) {
     socket.emit(`session-done-${sesid}`);
-    this.dispatchMessage(sesid, 'Query execution finished');
+    if (!props.skipFinishedMessage) {
+      this.dispatchMessage(sesid, 'Query execution finished');
+    }
+    const session = this.opened.find(x => x.sesid == sesid);
+    if (session.loadingReader_jslid) {
+      socket.emit(`session-jslid-done-${session.loadingReader_jslid}`);
+    }
+    if (session.killOnDone) {
+      this.kill({ sesid });
+    }
   },
 
   handle_recordset(sesid, props) {
@@ -59,13 +71,24 @@ module.exports = {
     jsldata.notifyChangedStats(stats);
   },
 
+  handle_initializeFile(sesid, props) {
+    const { jslid } = props;
+    socket.emit(`session-initialize-file-${jslid}`);
+  },
+
   handle_ping() {},
 
-  create_meta: 'post',
+  create_meta: true,
   async create({ conid, database }) {
     const sesid = uuidv1();
-    const connection = await connections.get({ conid });
-    const subprocess = fork(process.argv[1], ['sessionProcess', ...process.argv.slice(3)]);
+    const connection = await connections.getCore({ conid });
+    const subprocess = fork(global['API_PACKAGE'] || process.argv[1], [
+      '--is-forked-api',
+      '--start-process',
+      'sessionProcess',
+      ...processArgs.getPassArgs(),
+      // ...process.argv.slice(3),
+    ]);
     const newOpened = {
       conid,
       database,
@@ -81,10 +104,10 @@ module.exports = {
       this[`handle_${msgtype}`](sesid, message);
     });
     subprocess.send({ msgtype: 'connect', ...connection, database });
-    return newOpened;
+    return _.pick(newOpened, ['conid', 'database', 'sesid']);
   },
 
-  executeQuery_meta: 'post',
+  executeQuery_meta: true,
   async executeQuery({ sesid, sql }) {
     const session = this.opened.find(x => x.sesid == sesid);
     if (!session) {
@@ -98,7 +121,30 @@ module.exports = {
     return { state: 'ok' };
   },
 
-  // cancel_meta: 'post',
+  executeReader_meta: true,
+  async executeReader({ conid, database, sql, queryName, appFolder }) {
+    const { sesid } = await this.create({ conid, database });
+    const session = this.opened.find(x => x.sesid == sesid);
+    session.killOnDone = true;
+    const jslid = uuidv1();
+    session.loadingReader_jslid = jslid;
+    const fileName = queryName && appFolder ? path.join(appdir(), appFolder, `${queryName}.query.sql`) : null;
+
+    session.subprocess.send({ msgtype: 'executeReader', sql, fileName, jslid });
+
+    return { jslid };
+  },
+
+  stopLoadingReader_meta: true,
+  async stopLoadingReader({ jslid }) {
+    const session = this.opened.find(x => x.loadingReader_jslid == jslid);
+    if (session) {
+      this.kill({ sesid: session.sesid });
+    }
+    return true;
+  },
+
+  // cancel_meta: true,
   // async cancel({ sesid }) {
   //   const session = this.opened.find((x) => x.sesid == sesid);
   //   if (!session) {
@@ -108,7 +154,7 @@ module.exports = {
   //   return { state: 'ok' };
   // },
 
-  kill_meta: 'post',
+  kill_meta: true,
   async kill({ sesid }) {
     const session = this.opened.find(x => x.sesid == sesid);
     if (!session) {
@@ -119,7 +165,7 @@ module.exports = {
     return { state: 'ok' };
   },
 
-  // runCommand_meta: 'post',
+  // runCommand_meta: true,
   // async runCommand({ conid, database, sql }) {
   //   console.log(`Running SQL command , conid=${conid}, database=${database}, sql=${sql}`);
   //   const opened = await this.ensureOpened(conid, database);

packages/api/src/controllers/uploads.js

@@ -25,4 +25,12 @@ module.exports = {
       });
     });
   },
+
+  get_meta: {
+    method: 'get',
+    raw: true,
+  },
+  get(req, res) {
+    res.sendFile(path.join(uploadsdir(), req.query.file));
+  },
 };

packages/api/src/currentVersion.js

@@ -1,5 +1,5 @@
 
 module.exports = { 
-  version: '3.9.5',
-  buildTime: '2021-02-08T18:21:44.182Z'
+  version: '4.1.1',
+  buildTime: '2021-04-17T07:22:49.702Z'
 };

packages/api/src/index.js

@@ -1,15 +1,17 @@
 const shell = require('./shell');
+const processArgs = require('./utility/processArgs');
+const dbgateTools = require('dbgate-tools');
 
-const argument = process.argv[2];
-if (argument && argument.endsWith('Process')) {
+global['DBGATE_TOOLS'] = dbgateTools;
+
+if (processArgs.startProcess) {
   const proc = require('./proc');
-
-  const module = proc[argument];
+  const module = proc[processArgs.startProcess];
   module.start();
-} else if (!module['parent'] && !process.argv.includes('--checkParent')) {
+} else if (!processArgs.checkParent && !global['API_PACKAGE']) {
   const main = require('./main');
 
-  main.start(argument);
+  main.start();
 }
 
 module.exports = {

packages/api/src/main.js

@@ -4,10 +4,7 @@ const bodyParser = require('body-parser');
 const fileUpload = require('express-fileupload');
 const http = require('http');
 const cors = require('cors');
-const io = require('socket.io');
-const fs = require('fs');
-const findFreePort = require('find-free-port');
-const childProcessChecker = require('./utility/childProcessChecker');
+const getPort = require('get-port');
 const path = require('path');
 
 const useController = require('./utility/useController');
@@ -22,27 +19,31 @@ const runners = require('./controllers/runners');
 const jsldata = require('./controllers/jsldata');
 const config = require('./controllers/config');
 const archive = require('./controllers/archive');
+const apps = require('./controllers/apps');
 const uploads = require('./controllers/uploads');
 const plugins = require('./controllers/plugins');
 const files = require('./controllers/files');
 const scheduler = require('./controllers/scheduler');
+const queryHistory = require('./controllers/queryHistory');
 
 const { rundir } = require('./utility/directories');
+const platformInfo = require('./utility/platformInfo');
+const getExpressPath = require('./utility/getExpressPath');
+const { getLogins } = require('./utility/hasPermission');
+const _ = require('lodash');
 
-function start(argument = null) {
+function start() {
   // console.log('process.argv', process.argv);
 
   const app = express();
 
   const server = http.createServer(app);
-  socket.set(io(server));
 
-  if (process.env.LOGIN && process.env.PASSWORD) {
+  const logins = getLogins();
+  if (logins) {
     app.use(
       basicAuth({
-        users: {
-          [process.env.LOGIN]: process.env.PASSWORD,
-        },
+        users: _.fromPairs(logins.map(x => [x.login, x.password])),
         challenge: true,
         realm: 'DbGate Web App',
       })
@@ -50,65 +51,112 @@ function start(argument = null) {
   }
 
   app.use(cors());
+
+  app.get(getExpressPath('/stream'), async function (req, res) {
+    res.set({
+      'Cache-Control': 'no-cache',
+      'Content-Type': 'text/event-stream',
+      'X-Accel-Buffering': 'no',
+      Connection: 'keep-alive',
+    });
+    res.flushHeaders();
+
+    // Tell the client to retry every 10 seconds if connectivity is lost
+    res.write('retry: 10000\n\n');
+    socket.setSseResponse(res);
+  });
+
   app.use(bodyParser.json({ limit: '50mb' }));
 
   app.use(
-    '/uploads',
+    getExpressPath('/uploads'),
     fileUpload({
       limits: { fileSize: 4 * 1024 * 1024 },
     })
   );
 
-  useController(app, '/connections', connections);
-  useController(app, '/server-connections', serverConnections);
-  useController(app, '/database-connections', databaseConnections);
-  useController(app, '/metadata', metadata);
-  useController(app, '/sessions', sessions);
-  useController(app, '/runners', runners);
-  useController(app, '/jsldata', jsldata);
-  useController(app, '/config', config);
-  useController(app, '/archive', archive);
-  useController(app, '/uploads', uploads);
-  useController(app, '/plugins', plugins);
-  useController(app, '/files', files);
-  useController(app, '/scheduler', scheduler);
+  useAllControllers(app, null);
 
   // if (process.env.PAGES_DIRECTORY) {
   //   app.use('/pages', express.static(process.env.PAGES_DIRECTORY));
   // }
 
-  app.use('/runners/data', express.static(rundir()));
+  app.use(getExpressPath('/runners/data'), express.static(rundir()));
 
-  if (fs.existsSync('/home/dbgate-docker/build')) {
+  if (platformInfo.isDocker) {
     // server static files inside docker container
-    app.use(express.static('/home/dbgate-docker/build'));
-  } else {
-    if (argument != 'startNodeWeb') {
-      app.get('/', (req, res) => {
-        res.send('DbGate API');
-      });
-    }
-  }
+    app.use(getExpressPath('/'), express.static('/home/dbgate-docker/public'));
 
-  if (argument == '--dynport') {
-    childProcessChecker();
-
-    findFreePort(53911, function (err, port) {
+    const port = process.env.PORT || 3000;
+    console.log('DbGate API listening on port (docker build)', port);
+    server.listen(port);
+  } else if (platformInfo.isNpmDist) {
+    app.use(getExpressPath('/'), express.static(path.join(__dirname, '../../dbgate-web/public')));
+    getPort({
+      port: parseInt(
+        // @ts-ignore
+        process.env.PORT || 3000
+      ),
+    }).then(port => {
       server.listen(port, () => {
-        console.log(`DbGate API listening on port ${port}`);
-        process.send({ msgtype: 'listening', port });
-      });
-    });
-  } else if (argument == 'startNodeWeb') {
-    app.use(express.static(path.join(__dirname, '../../dbgate-web/build')));
-    findFreePort(5000, function (err, port) {
-      server.listen(port, () => {
-        console.log(`DbGate API listening on port ${port}`);
+        console.log(`DbGate API listening on port ${port} (NPM build)`);
       });
     });
+  } else if (process.env.DEVWEB) {
+    console.log('__dirname', __dirname);
+    console.log(path.join(__dirname, '../../web/public/build'));
+    app.use(getExpressPath('/'), express.static(path.join(__dirname, '../../web/public')));
+
+    const port = process.env.PORT || 3000;
+    console.log('DbGate API & web listening on port (dev web build)', port);
+    server.listen(port);
   } else {
-    server.listen(3000);
+    app.get(getExpressPath('/'), (req, res) => {
+      res.send('DbGate API');
+    });
+
+    const port = process.env.PORT || 3000;
+    console.log('DbGate API listening on port (dev API build)', port);
+    server.listen(port);
   }
+
+  function shutdown() {
+    console.log('\nShutting down DbGate API server');
+    server.close(() => {
+      console.log('Server shut down, terminating');
+      process.exit(0);
+    });
+    setTimeout(() => {
+      console.log('Server close timeout, terminating');
+      process.exit(0);
+    }, 1000);
+  }
+
+  process.on('SIGINT', shutdown);
+  process.on('SIGTERM', shutdown);
+  process.on('SIGBREAK', shutdown);
 }
 
-module.exports = { start };
+function useAllControllers(app, electron) {
+  useController(app, electron, '/connections', connections);
+  useController(app, electron, '/server-connections', serverConnections);
+  useController(app, electron, '/database-connections', databaseConnections);
+  useController(app, electron, '/metadata', metadata);
+  useController(app, electron, '/sessions', sessions);
+  useController(app, electron, '/runners', runners);
+  useController(app, electron, '/jsldata', jsldata);
+  useController(app, electron, '/config', config);
+  useController(app, electron, '/archive', archive);
+  useController(app, electron, '/uploads', uploads);
+  useController(app, electron, '/plugins', plugins);
+  useController(app, electron, '/files', files);
+  useController(app, electron, '/scheduler', scheduler);
+  useController(app, electron, '/query-history', queryHistory);
+  useController(app, electron, '/apps', apps);
+}
+
+function initializeElectronSender(electronSender) {
+  socket.setElectronSender(electronSender);
+}
+
+module.exports = { start, useAllControllers, initializeElectronSender, configController: config };

packages/api/src/nativeModules.js

@@ -1,5 +1,13 @@
 const argIndex = process.argv.indexOf('--native-modules');
-const redirectFile = argIndex > 0 ? process.argv[argIndex + 1] : null;
+const redirectFile = global['NATIVE_MODULES'] || (argIndex > 0 ? process.argv[argIndex + 1] : null);
 
-// @ts-ignore
-module.exports = redirectFile ? __non_webpack_require__(redirectFile) : require('./nativeModulesContent');
+function requireDynamic(file) {
+  try {
+    // @ts-ignore
+    return __non_webpack_require__(redirectFile);
+  } catch (err) {
+    return require(redirectFile);
+  }
+}
+
+module.exports = redirectFile ? requireDynamic(redirectFile) : require('./nativeModulesContent');

packages/api/src/proc/connectProcess.js

@@ -2,6 +2,17 @@ const childProcessChecker = require('../utility/childProcessChecker');
 const requireEngineDriver = require('../utility/requireEngineDriver');
 const connectUtility = require('../utility/connectUtility');
 const { handleProcessCommunication } = require('../utility/processComm');
+const { pickSafeConnectionInfo } = require('../utility/crypting');
+const _ = require('lodash');
+
+const formatErrorDetail = (e, connection) => `${e.stack}
+
+Error JSON: ${JSON.stringify(e, undefined, 2)}
+
+Connection: ${JSON.stringify(pickSafeConnectionInfo(connection), undefined, 2)}
+
+Platform: ${process.platform}
+`;
 
 function start() {
   childProcessChecker();
@@ -9,12 +20,16 @@ function start() {
     if (handleProcessCommunication(connection)) return;
     try {
       const driver = requireEngineDriver(connection);
-      const conn = await connectUtility(driver, connection);
+      const conn = await connectUtility(driver, connection, 'app');
       const res = await driver.getVersion(conn);
       process.send({ msgtype: 'connected', ...res });
     } catch (e) {
       console.error(e);
-      process.send({ msgtype: 'error', error: e.message });
+      process.send({
+        msgtype: 'error',
+        error: e.message,
+        detail: formatErrorDetail(e, connection),
+      });
     }
   });
 }

packages/api/src/proc/databaseConnectionProcess.js

@@ -1,15 +1,29 @@
 const stableStringify = require('json-stable-stringify');
+const { splitQuery } = require('dbgate-query-splitter');
 const childProcessChecker = require('../utility/childProcessChecker');
+const { extractBoolSettingsValue, extractIntSettingsValue } = require('dbgate-tools');
 const requireEngineDriver = require('../utility/requireEngineDriver');
 const connectUtility = require('../utility/connectUtility');
 const { handleProcessCommunication } = require('../utility/processComm');
+const { SqlGenerator } = require('dbgate-tools');
+const generateDeploySql = require('../shell/generateDeploySql');
+const { dumpSqlSelect } = require('dbgate-sqltree');
 
 let systemConnection;
 let storedConnection;
 let afterConnectCallbacks = [];
+let afterAnalyseCallbacks = [];
 let analysedStructure = null;
 let lastPing = null;
 let lastStatus = null;
+let analysedTime = 0;
+let serverVersion;
+
+let statusCounter = 0;
+function getStatusCounter() {
+  statusCounter += 1;
+  return statusCounter;
+}
 
 async function checkedAsyncCall(promise) {
   try {
@@ -26,27 +40,53 @@ async function checkedAsyncCall(promise) {
   }
 }
 
+let loadingModel = false;
+
 async function handleFullRefresh() {
+  loadingModel = true;
   const driver = requireEngineDriver(storedConnection);
-  analysedStructure = await checkedAsyncCall(driver.analyseFull(systemConnection));
+  setStatusName('loadStructure');
+  analysedStructure = await checkedAsyncCall(driver.analyseFull(systemConnection, serverVersion));
+  analysedTime = new Date().getTime();
   process.send({ msgtype: 'structure', structure: analysedStructure });
+  process.send({ msgtype: 'structureTime', analysedTime });
   setStatusName('ok');
+
+  loadingModel = false;
+  resolveAnalysedPromises();
 }
 
-async function handleIncrementalRefresh() {
+async function handleIncrementalRefresh(forceSend) {
+  loadingModel = true;
   const driver = requireEngineDriver(storedConnection);
-  const newStructure = await checkedAsyncCall(driver.analyseIncremental(systemConnection, analysedStructure));
+  setStatusName('checkStructure');
+  const newStructure = await checkedAsyncCall(
+    driver.analyseIncremental(systemConnection, analysedStructure, serverVersion)
+  );
+  analysedTime = new Date().getTime();
   if (newStructure != null) {
     analysedStructure = newStructure;
+  }
+
+  if (forceSend || newStructure != null) {
     process.send({ msgtype: 'structure', structure: analysedStructure });
   }
+
+  process.send({ msgtype: 'structureTime', analysedTime });
   setStatusName('ok');
+  loadingModel = false;
+  resolveAnalysedPromises();
+}
+
+function handleSyncModel() {
+  if (loadingModel) return;
+  handleIncrementalRefresh();
 }
 
 function setStatus(status) {
   const statusString = stableStringify(status);
   if (lastStatus != statusString) {
-    process.send({ msgtype: 'status', status });
+    process.send({ msgtype: 'status', status: { ...status, counter: getStatusCounter() } });
     lastStatus = statusString;
   }
 }
@@ -55,20 +95,35 @@ function setStatusName(name) {
   setStatus({ name });
 }
 
-async function handleConnect({ connection, structure }) {
+async function readVersion() {
+  const driver = requireEngineDriver(storedConnection);
+  const version = await driver.getVersion(systemConnection);
+  process.send({ msgtype: 'version', version });
+  serverVersion = version;
+}
+
+async function handleConnect({ connection, structure, globalSettings }) {
   storedConnection = connection;
   lastPing = new Date().getTime();
 
   if (!structure) setStatusName('pending');
   const driver = requireEngineDriver(storedConnection);
-  systemConnection = await checkedAsyncCall(connectUtility(driver, storedConnection));
+  systemConnection = await checkedAsyncCall(connectUtility(driver, storedConnection, 'app'));
+  await checkedAsyncCall(readVersion());
   if (structure) {
     analysedStructure = structure;
-    handleIncrementalRefresh();
+    handleIncrementalRefresh(true);
   } else {
     handleFullRefresh();
   }
-  setInterval(handleIncrementalRefresh, 30 * 1000);
+
+  if (extractBoolSettingsValue(globalSettings, 'connection.autoRefresh', false)) {
+    setInterval(
+      handleIncrementalRefresh,
+      extractIntSettingsValue(globalSettings, 'connection.autoRefreshInterval', 30, 3, 3600) * 1000
+    );
+  }
+
   for (const [resolve] of afterConnectCallbacks) {
     resolve();
   }
@@ -82,10 +137,37 @@ function waitConnected() {
   });
 }
 
+function waitStructure() {
+  if (analysedStructure) return Promise.resolve();
+  return new Promise((resolve, reject) => {
+    afterAnalyseCallbacks.push([resolve, reject]);
+  });
+}
+
+function resolveAnalysedPromises() {
+  for (const [resolve] of afterAnalyseCallbacks) {
+    resolve();
+  }
+  afterAnalyseCallbacks = [];
+}
+
+async function handleRunScript({ msgid, sql }) {
+  await waitConnected();
+  const driver = requireEngineDriver(storedConnection);
+  try {
+    ensureExecuteCustomScript(driver);
+    await driver.script(systemConnection, sql);
+    process.send({ msgtype: 'response', msgid });
+  } catch (err) {
+    process.send({ msgtype: 'response', msgid, errorMessage: err.message });
+  }
+}
+
 async function handleQueryData({ msgid, sql }) {
   await waitConnected();
   const driver = requireEngineDriver(storedConnection);
   try {
+    ensureExecuteCustomScript(driver);
     const res = await driver.query(systemConnection, sql);
     process.send({ msgtype: 'response', msgid, ...res });
   } catch (err) {
@@ -93,6 +175,111 @@ async function handleQueryData({ msgid, sql }) {
   }
 }
 
+async function handleSqlSelect({ msgid, select }) {
+  const driver = requireEngineDriver(storedConnection);
+  const dmp = driver.createDumper();
+  dumpSqlSelect(dmp, select);
+  return handleQueryData({ msgid, sql: dmp.s });
+}
+
+async function handleDriverDataCore(msgid, callMethod) {
+  await waitConnected();
+  const driver = requireEngineDriver(storedConnection);
+  try {
+    const result = await callMethod(driver);
+    process.send({ msgtype: 'response', msgid, result });
+  } catch (err) {
+    process.send({ msgtype: 'response', msgid, errorMessage: err.message });
+  }
+}
+
+async function handleCollectionData({ msgid, options }) {
+  return handleDriverDataCore(msgid, driver => driver.readCollection(systemConnection, options));
+}
+
+async function handleLoadKeys({ msgid, root }) {
+  return handleDriverDataCore(msgid, driver => driver.loadKeys(systemConnection, root));
+}
+
+async function handleLoadKeyInfo({ msgid, key }) {
+  return handleDriverDataCore(msgid, driver => driver.loadKeyInfo(systemConnection, key));
+}
+
+async function handleCallMethod({ msgid, method, args }) {
+  return handleDriverDataCore(msgid, driver => {
+    ensureExecuteCustomScript(driver);
+    return driver.callMethod(systemConnection, method, args);
+  });
+}
+
+async function handleLoadKeyTableRange({ msgid, key, cursor, count }) {
+  return handleDriverDataCore(msgid, driver => driver.loadKeyTableRange(systemConnection, key, cursor, count));
+}
+
+async function handleLoadFieldValues({ msgid, schemaName, pureName, field, search }) {
+  return handleDriverDataCore(msgid, driver =>
+    driver.loadFieldValues(systemConnection, { schemaName, pureName }, field, search)
+  );
+}
+
+function ensureExecuteCustomScript(driver) {
+  if (driver.readOnlySessions) {
+    return;
+  }
+  if (storedConnection.isReadOnly) {
+    throw new Error('Connection is read only');
+  }
+}
+
+async function handleUpdateCollection({ msgid, changeSet }) {
+  await waitConnected();
+  const driver = requireEngineDriver(storedConnection);
+  try {
+    ensureExecuteCustomScript(driver);
+    const result = await driver.updateCollection(systemConnection, changeSet);
+    process.send({ msgtype: 'response', msgid, result });
+  } catch (err) {
+    process.send({ msgtype: 'response', msgid, errorMessage: err.message });
+  }
+}
+
+async function handleSqlPreview({ msgid, objects, options }) {
+  await waitStructure();
+  const driver = requireEngineDriver(storedConnection);
+
+  try {
+    const dmp = driver.createDumper();
+    const generator = new SqlGenerator(analysedStructure, options, objects, dmp, driver, systemConnection);
+
+    await generator.dump();
+    process.send({ msgtype: 'response', msgid, sql: dmp.s, isTruncated: generator.isTruncated });
+    if (generator.isUnhandledException) {
+      setTimeout(() => {
+        console.log('Exiting because of unhandled exception');
+        process.exit(0);
+      }, 500);
+    }
+  } catch (err) {
+    process.send({ msgtype: 'response', msgid, isError: true, errorMessage: err.message });
+  }
+}
+
+async function handleGenerateDeploySql({ msgid, modelFolder }) {
+  await waitStructure();
+
+  try {
+    const res = await generateDeploySql({
+      systemConnection,
+      connection: storedConnection,
+      analysedStructure,
+      modelFolder,
+    });
+    process.send({ ...res, msgtype: 'response', msgid });
+  } catch (err) {
+    process.send({ msgtype: 'response', msgid, isError: true, errorMessage: err.message });
+  }
+}
+
 // async function handleRunCommand({ msgid, sql }) {
 //   await waitConnected();
 //   const driver = engines(storedConnection);
@@ -107,7 +294,19 @@ function handlePing() {
 const messageHandlers = {
   connect: handleConnect,
   queryData: handleQueryData,
+  runScript: handleRunScript,
+  updateCollection: handleUpdateCollection,
+  collectionData: handleCollectionData,
+  loadKeys: handleLoadKeys,
+  loadKeyInfo: handleLoadKeyInfo,
+  callMethod: handleCallMethod,
+  loadKeyTableRange: handleLoadKeyTableRange,
+  sqlPreview: handleSqlPreview,
   ping: handlePing,
+  syncModel: handleSyncModel,
+  generateDeploySql: handleGenerateDeploySql,
+  loadFieldValues: handleLoadFieldValues,
+  sqlSelect: handleSqlSelect,
   // runCommand: handleRunCommand,
 };
 
@@ -132,6 +331,7 @@ function start() {
     try {
       await handleMessage(message);
     } catch (e) {
+      console.error('Error in DB connection', e);
       process.send({ msgtype: 'error', error: e.message });
     }
   });

packages/api/src/proc/serverConnectionProcess.js

@@ -1,4 +1,5 @@
 const stableStringify = require('json-stable-stringify');
+const { extractBoolSettingsValue, extractIntSettingsValue } = require('dbgate-tools');
 const childProcessChecker = require('../utility/childProcessChecker');
 const requireEngineDriver = require('../utility/requireEngineDriver');
 const { decryptConnection } = require('../utility/crypting');
@@ -31,6 +32,12 @@ async function handleRefresh() {
   }
 }
 
+async function readVersion() {
+  const driver = requireEngineDriver(storedConnection);
+  const version = await driver.getVersion(systemConnection);
+  process.send({ msgtype: 'version', version });
+}
+
 function setStatus(status) {
   const statusString = stableStringify(status);
   if (lastStatus != statusString) {
@@ -45,14 +52,21 @@ function setStatusName(name) {
 
 async function handleConnect(connection) {
   storedConnection = connection;
+  const { globalSettings } = storedConnection;
   setStatusName('pending');
   lastPing = new Date().getTime();
 
   const driver = requireEngineDriver(storedConnection);
   try {
-    systemConnection = await connectUtility(driver, storedConnection);
+    systemConnection = await connectUtility(driver, storedConnection, 'app');
+    readVersion();
     handleRefresh();
-    setInterval(handleRefresh, 30 * 1000);
+    if (extractBoolSettingsValue(globalSettings, 'connection.autoRefresh', false)) {
+      setInterval(
+        handleRefresh,
+        extractIntSettingsValue(globalSettings, 'connection.autoRefreshInterval', 30, 5, 3600) * 1000
+      );
+    }
   } catch (err) {
     setStatus({
       name: 'error',
@@ -69,9 +83,13 @@ function handlePing() {
 
 async function handleCreateDatabase({ name }) {
   const driver = requireEngineDriver(storedConnection);
-  systemConnection = await connectUtility(driver, storedConnection);
+  systemConnection = await connectUtility(driver, storedConnection, 'app');
   console.log(`RUNNING SCRIPT: CREATE DATABASE ${driver.dialect.quoteIdentifier(name)}`);
-  await driver.query(systemConnection, `CREATE DATABASE ${driver.dialect.quoteIdentifier(name)}`);
+  if (driver.createDatabase) {
+    await driver.createDatabase(systemConnection, name);
+  } else {
+    await driver.query(systemConnection, `CREATE DATABASE ${driver.dialect.quoteIdentifier(name)}`);
+  }
   await handleRefresh();
 }
 

packages/api/src/proc/sessionProcess.js

@@ -3,7 +3,7 @@ const path = require('path');
 const fs = require('fs');
 const _ = require('lodash');
 const childProcessChecker = require('../utility/childProcessChecker');
-const goSplit = require('../utility/goSplit');
+const { splitQuery } = require('dbgate-query-splitter');
 
 const { jsldir } = require('../utility/directories');
 const requireEngineDriver = require('../utility/requireEngineDriver');
@@ -17,22 +17,40 @@ let afterConnectCallbacks = [];
 // let currentHandlers = [];
 
 class TableWriter {
-  constructor(columns, resultIndex) {
-    this.jslid = uuidv1();
-    this.currentFile = path.join(jsldir(), `${this.jslid}.jsonl`);
+  constructor() {
     this.currentRowCount = 0;
     this.currentChangeIndex = 1;
-    fs.writeFileSync(this.currentFile, JSON.stringify({ columns }) + '\n');
+    this.initializedFile = false;
+  }
+
+  initializeFromQuery(structure, resultIndex) {
+    this.jslid = uuidv1();
+    this.currentFile = path.join(jsldir(), `${this.jslid}.jsonl`);
+    fs.writeFileSync(
+      this.currentFile,
+      JSON.stringify({
+        ...structure,
+        __isStreamHeader: true,
+      }) + '\n'
+    );
     this.currentStream = fs.createWriteStream(this.currentFile, { flags: 'a' });
     this.writeCurrentStats(false, false);
     this.resultIndex = resultIndex;
+    this.initializedFile = true;
     process.send({ msgtype: 'recordset', jslid: this.jslid, resultIndex });
   }
 
+  initializeFromReader(jslid) {
+    this.jslid = jslid;
+    this.currentFile = path.join(jsldir(), `${this.jslid}.jsonl`);
+    this.writeCurrentStats(false, false);
+  }
+
   row(row) {
     // console.log('ACCEPT ROW', row);
     this.currentStream.write(JSON.stringify(row) + '\n');
     this.currentRowCount += 1;
+
     if (!this.plannedStats) {
       this.plannedStats = true;
       process.nextTick(() => {
@@ -43,6 +61,21 @@ class TableWriter {
     }
   }
 
+  rowFromReader(row) {
+    if (!this.initializedFile) {
+      process.send({ msgtype: 'initializeFile', jslid: this.jslid });
+      this.initializedFile = true;
+
+      fs.writeFileSync(this.currentFile, JSON.stringify(row) + '\n');
+      this.currentStream = fs.createWriteStream(this.currentFile, { flags: 'a' });
+      this.writeCurrentStats(false, false);
+      this.initializedFile = true;
+      return;
+    }
+
+    this.row(row);
+  }
+
   writeCurrentStats(isFinished = false, emitEvent = false) {
     const stats = {
       rowCount: this.currentRowCount,
@@ -57,10 +90,11 @@ class TableWriter {
     }
   }
 
-  close() {
+  close(afterClose) {
     if (this.currentStream) {
       this.currentStream.end(() => {
         this.writeCurrentStats(true, true);
+        if (afterClose) afterClose();
       });
     }
   }
@@ -92,7 +126,11 @@ class StreamHandler {
 
   recordset(columns) {
     this.closeCurrentWriter();
-    this.currentWriter = new TableWriter(columns, this.resultIndexHolder.value);
+    this.currentWriter = new TableWriter();
+    this.currentWriter.initializeFromQuery(
+      Array.isArray(columns) ? { columns } : columns,
+      this.resultIndexHolder.value
+    );
     this.resultIndexHolder.value += 1;
 
     // this.writeCurrentStats();
@@ -104,7 +142,6 @@ class StreamHandler {
     // }, 500);
   }
   row(row) {
-    // console.log('ACCEPT ROW', row);
     if (this.currentWriter) this.currentWriter.row(row);
     else if (row.message) process.send({ msgtype: 'info', info: { message: row.message } });
     // this.onRow(this.jslid);
@@ -129,11 +166,22 @@ function handleStream(driver, resultIndexHolder, sql) {
   });
 }
 
+function allowExecuteCustomScript(driver) {
+  if (driver.readOnlySessions) {
+    return true;
+  }
+  if (storedConnection.isReadOnly) {
+    return false;
+    // throw new Error('Connection is read only');
+  }
+  return true;
+}
+
 async function handleConnect(connection) {
   storedConnection = connection;
 
   const driver = requireEngineDriver(storedConnection);
-  systemConnection = await connectUtility(driver, storedConnection);
+  systemConnection = await connectUtility(driver, storedConnection, 'app');
   for (const [resolve] of afterConnectCallbacks) {
     resolve();
   }
@@ -157,10 +205,23 @@ async function handleExecuteQuery({ sql }) {
   await waitConnected();
   const driver = requireEngineDriver(storedConnection);
 
+  if (!allowExecuteCustomScript(driver)) {
+    process.send({
+      msgtype: 'info',
+      info: {
+        message: 'Connection without read-only sessions is read only',
+        severity: 'error',
+      },
+    });
+    process.send({ msgtype: 'done', skipFinishedMessage: true });
+    return;
+    //process.send({ msgtype: 'error', error: e.message });
+  }
+
   const resultIndexHolder = {
     value: 0,
   };
-  for (const sqlItem of goSplit(sql)) {
+  for (const sqlItem of splitQuery(sql, driver.getQuerySplitterOptions('stream'))) {
     await handleStream(driver, resultIndexHolder, sqlItem);
     // const handler = new StreamHandler(resultIndex);
     // const stream = await driver.stream(systemConnection, sqlItem, handler);
@@ -170,9 +231,39 @@ async function handleExecuteQuery({ sql }) {
   process.send({ msgtype: 'done' });
 }
 
+async function handleExecuteReader({ jslid, sql, fileName }) {
+  await waitConnected();
+
+  const driver = requireEngineDriver(storedConnection);
+
+  if (fileName) {
+    sql = fs.readFileSync(fileName, 'utf-8');
+  } else {
+    if (!allowExecuteCustomScript(driver)) {
+      process.send({ msgtype: 'done' });
+      return;
+    }
+  }
+
+  const writer = new TableWriter();
+  writer.initializeFromReader(jslid);
+
+  const reader = await driver.readQuery(systemConnection, sql);
+
+  reader.on('data', data => {
+    writer.rowFromReader(data);
+  });
+  reader.on('end', () => {
+    writer.close(() => {
+      process.send({ msgtype: 'done' });
+    });
+  });
+}
+
 const messageHandlers = {
   connect: handleConnect,
   executeQuery: handleExecuteQuery,
+  executeReader: handleExecuteReader,
   // cancel: handleCancel,
 };
 

packages/api/src/shell/archiveReader.js

@@ -1,9 +1,9 @@
 const path = require('path');
-const { archivedir } = require('../utility/directories');
+const { archivedir, resolveArchiveFolder } = require('../utility/directories');
 const jsonLinesReader = require('./jsonLinesReader');
 
 function archiveReader({ folderName, fileName, ...other }) {
-  const jsonlFile = path.join(archivedir(), folderName, `${fileName}.jsonl`);
+  const jsonlFile = path.join(resolveArchiveFolder(folderName), `${fileName}.jsonl`);
   const res = jsonLinesReader({ fileName: jsonlFile, ...other });
   return res;
 }

packages/api/src/shell/archiveWriter.js

@@ -1,11 +1,11 @@
 const path = require('path');
 const fs = require('fs');
-const { archivedir } = require('../utility/directories');
+const { archivedir, resolveArchiveFolder } = require('../utility/directories');
 // const socket = require('../utility/socket');
 const jsonLinesWriter = require('./jsonLinesWriter');
 
 function archiveWriter({ folderName, fileName }) {
-  const dir = path.join(archivedir(), folderName);
+  const dir = resolveArchiveFolder(folderName);
   if (!fs.existsSync(dir)) {
     console.log(`Creating directory ${dir}`);
     fs.mkdirSync(dir);

packages/api/src/shell/copyStream.js

@@ -1,9 +1,47 @@
-function copyStream(input, output) {
+const EnsureStreamHeaderStream = require('../utility/EnsureStreamHeaderStream');
+const Stream = require('stream');
+const ColumnMapTransformStream = require('../utility/ColumnMapTransformStream');
+
+function copyStream(input, output, options) {
+  const { columns } = options || {};
+
+  const transforms = [];
+  if (columns) {
+    transforms.push(new ColumnMapTransformStream(columns));
+  }
+  if (output.requireFixedStructure) {
+    transforms.push(new EnsureStreamHeaderStream());
+  }
+
+  // return new Promise((resolve, reject) => {
+  //   Stream.pipeline(input, ...transforms, output, err => {
+  //     if (err) {
+  //       reject(err);
+  //     } else {
+  //       resolve();
+  //     }
+  //   });
+  // });
+
   return new Promise((resolve, reject) => {
     const finisher = output['finisher'] || output;
     finisher.on('finish', resolve);
     finisher.on('error', reject);
-    input.pipe(output);
+
+    let lastStream = input;
+    for (const tran of transforms) {
+      lastStream.pipe(tran);
+      lastStream = tran;
+    }
+    lastStream.pipe(output);
+
+    //  if (output.requireFixedStructure) {
+    //   const ensureHeader = new EnsureStreamHeaderStream();
+    //   input.pipe(ensureHeader);
+    //   ensureHeader.pipe(output);
+    // } else {
+    //   input.pipe(output);
+    // }
   });
 }
 

packages/api/src/shell/deployDb.js

@@ -0,0 +1,17 @@
+const generateDeploySql = require('./generateDeploySql');
+const executeQuery = require('./executeQuery');
+
+async function deployDb({ connection, systemConnection, driver, analysedStructure, modelFolder, loadedDbModel }) {
+  const { sql } = await generateDeploySql({
+    connection,
+    systemConnection,
+    driver,
+    analysedStructure,
+    modelFolder,
+    loadedDbModel,
+  });
+  // console.log('RUNNING DEPLOY SCRIPT:', sql);
+  await executeQuery({ connection, systemConnection, driver, sql });
+}
+
+module.exports = deployDb;

packages/api/src/shell/executeQuery.js

@@ -1,21 +1,14 @@
-const goSplit = require('../utility/goSplit');
 const requireEngineDriver = require('../utility/requireEngineDriver');
-const { decryptConnection } = require('../utility/crypting');
 const connectUtility = require('../utility/connectUtility');
 
-async function executeQuery({ connection, sql }) {
+async function executeQuery({ connection = undefined, systemConnection = undefined, driver = undefined, sql }) {
   console.log(`Execute query ${sql}`);
 
-  const driver = requireEngineDriver(connection);
-  const pool = await connectUtility(driver, connection);
+  if (!driver) driver = requireEngineDriver(connection);
+  const pool = systemConnection || (await connectUtility(driver, connection, 'script'));
   console.log(`Connected.`);
 
-  for (const sqlItem of goSplit(sql)) {
-    console.log('Executing query', sqlItem);
-    await driver.query(pool, sqlItem);
-  }
-
-  console.log(`Query finished`);
+  await driver.script(pool, sql);
 }
 
 module.exports = executeQuery;

packages/api/src/shell/fakeObjectReader.js

@@ -5,7 +5,7 @@ async function fakeObjectReader({ delay = 0 } = {}) {
     objectMode: true,
   });
   function doWrite() {
-    pass.write({ columns: [{ columnName: 'id' }, { columnName: 'country' }] });
+    pass.write({ columns: [{ columnName: 'id' }, { columnName: 'country' }], __isStreamHeader: true });
     pass.write({ id: 1, country: 'Czechia' });
     pass.write({ id: 2, country: 'Austria' });
     pass.write({ country: 'Germany', id: 3 });

packages/api/src/shell/generateDeploySql.js

@@ -0,0 +1,61 @@
+const {
+  getAlterDatabaseScript,
+  generateDbPairingId,
+  matchPairedObjects,
+  databaseInfoFromYamlModel,
+  extendDatabaseInfo,
+  modelCompareDbDiffOptions,
+  enrichWithPreloadedRows,
+} = require('dbgate-tools');
+const importDbModel = require('../utility/importDbModel');
+const requireEngineDriver = require('../utility/requireEngineDriver');
+const connectUtility = require('../utility/connectUtility');
+
+async function generateDeploySql({
+  connection,
+  systemConnection = undefined,
+  driver = undefined,
+  analysedStructure = undefined,
+  modelFolder = undefined,
+  loadedDbModel = undefined,
+}) {
+  if (!driver) driver = requireEngineDriver(connection);
+
+  const pool = systemConnection || (await connectUtility(driver, connection, 'read'));
+  if (!analysedStructure) {
+    analysedStructure = await driver.analyseFull(pool);
+  }
+
+  const deployedModel = generateDbPairingId(
+    extendDatabaseInfo(loadedDbModel ? databaseInfoFromYamlModel(loadedDbModel) : await importDbModel(modelFolder))
+  );
+  const currentModel = generateDbPairingId(extendDatabaseInfo(analysedStructure));
+  const opts = {
+    ...modelCompareDbDiffOptions,
+
+    noDropTable: true,
+    noDropColumn: true,
+    noDropConstraint: true,
+    noDropSqlObject: true,
+    noRenameTable: true,
+    noRenameColumn: true,
+  };
+  const currentModelPaired = matchPairedObjects(deployedModel, currentModel, opts);
+  const currentModelPairedPreloaded = await enrichWithPreloadedRows(deployedModel, currentModelPaired, pool, driver);
+
+  // console.log('currentModelPairedPreloaded', currentModelPairedPreloaded.tables[0]);
+  // console.log('deployedModel', deployedModel.tables[0]);
+  // console.log('currentModel', currentModel.tables[0]);
+  // console.log('currentModelPaired', currentModelPaired.tables[0]);
+  const res = getAlterDatabaseScript(
+    currentModelPairedPreloaded,
+    deployedModel,
+    opts,
+    currentModelPairedPreloaded,
+    deployedModel,
+    driver
+  );
+  return res;
+}
+
+module.exports = generateDeploySql;

packages/api/src/shell/index.js

@@ -6,7 +6,9 @@ const copyStream = require('./copyStream');
 const fakeObjectReader = require('./fakeObjectReader');
 const consoleObjectWriter = require('./consoleObjectWriter');
 const jsonLinesWriter = require('./jsonLinesWriter');
+const jsonArrayWriter = require('./jsonArrayWriter');
 const jsonLinesReader = require('./jsonLinesReader');
+const sqlDataWriter = require('./sqlDataWriter');
 const jslDataReader = require('./jslDataReader');
 const archiveWriter = require('./archiveWriter');
 const archiveReader = require('./archiveReader');
@@ -17,6 +19,7 @@ const requirePlugin = require('./requirePlugin');
 const download = require('./download');
 const executeQuery = require('./executeQuery');
 const loadFile = require('./loadFile');
+const deployDb = require('./deployDb');
 const initializeApiEnvironment = require('./initializeApiEnvironment');
 
 const dbgateApi = {
@@ -26,7 +29,9 @@ const dbgateApi = {
   tableReader,
   copyStream,
   jsonLinesWriter,
+  jsonArrayWriter,
   jsonLinesReader,
+  sqlDataWriter,
   fakeObjectReader,
   consoleObjectWriter,
   jslDataReader,
@@ -38,6 +43,7 @@ const dbgateApi = {
   registerPlugins,
   executeQuery,
   loadFile,
+  deployDb,
   initializeApiEnvironment,
 };
 

packages/api/src/shell/jsonArrayWriter.js

@@ -0,0 +1,49 @@
+const fs = require('fs');
+const stream = require('stream');
+
+class StringifyStream extends stream.Transform {
+  constructor() {
+    super({ objectMode: true });
+    this.wasHeader = false;
+    this.wasRecord = false;
+  }
+  _transform(chunk, encoding, done) {
+    let skip = false;
+
+    if (!this.wasHeader) {
+      skip = chunk.__isStreamHeader;
+      this.wasHeader = true;
+    }
+    if (!skip) {
+      if (!this.wasRecord) {
+        this.push('[\n');
+      } else {
+        this.push(',\n');
+      }
+      this.wasRecord = true;
+
+      this.push(JSON.stringify(chunk));
+    }
+    done();
+  }
+
+  _flush(done) {
+    if (!this.wasRecord) {
+      this.push('[]\n');
+    } else {
+      this.push('\n]\n');
+    }
+    done();
+  }
+}
+
+async function jsonArrayWriter({ fileName, encoding = 'utf-8' }) {
+  console.log(`Writing file ${fileName}`);
+  const stringify = new StringifyStream();
+  const fileStream = fs.createWriteStream(fileName, encoding);
+  stringify.pipe(fileStream);
+  stringify['finisher'] = fileStream;
+  return stringify;
+}
+
+module.exports = jsonArrayWriter;

packages/api/src/shell/jsonLinesReader.js

@@ -3,9 +3,8 @@ const stream = require('stream');
 const byline = require('byline');
 
 class ParseStream extends stream.Transform {
-  constructor({ header, limitRows }) {
+  constructor({ limitRows }) {
     super({ objectMode: true });
-    this.header = header;
     this.wasHeader = false;
     this.limitRows = limitRows;
     this.rowsWritten = 0;
@@ -13,7 +12,14 @@ class ParseStream extends stream.Transform {
   _transform(chunk, encoding, done) {
     const obj = JSON.parse(chunk);
     if (!this.wasHeader) {
-      if (!this.header) this.push({ columns: Object.keys(obj).map(columnName => ({ columnName })) });
+      if (!obj.__isStreamHeader) {
+        this.push({
+          __isStreamHeader: true,
+          __isDynamicStructure: true,
+          // columns: Object.keys(obj).map(columnName => ({ columnName })),
+        });
+      }
+
       this.wasHeader = true;
     }
     if (!this.limitRows || this.rowsWritten < this.limitRows) {
@@ -24,12 +30,12 @@ class ParseStream extends stream.Transform {
   }
 }
 
-async function jsonLinesReader({ fileName, encoding = 'utf-8', header = true, limitRows = undefined }) {
+async function jsonLinesReader({ fileName, encoding = 'utf-8', limitRows = undefined }) {
   console.log(`Reading file ${fileName}`);
 
   const fileStream = fs.createReadStream(fileName, encoding);
   const liner = byline(fileStream);
-  const parser = new ParseStream({ header, limitRows });
+  const parser = new ParseStream({ limitRows });
   liner.pipe(parser);
   return parser;
 }

packages/api/src/shell/jsonLinesWriter.js

@@ -8,10 +8,12 @@ class StringifyStream extends stream.Transform {
     this.wasHeader = false;
   }
   _transform(chunk, encoding, done) {
+    let skip = false;
     if (!this.wasHeader) {
-      if (this.header) this.push(JSON.stringify(chunk) + '\n');
+      skip = (chunk.__isStreamHeader && !this.header) || (chunk.__isStreamHeader && chunk.__isDynamicStructure);
       this.wasHeader = true;
-    } else {
+    }
+    if (!skip) {
       this.push(JSON.stringify(chunk) + '\n');
     }
     done();

packages/api/src/shell/queryReader.js

@@ -1,14 +1,26 @@
 const requireEngineDriver = require('../utility/requireEngineDriver');
-const { decryptConnection } = require('../utility/crypting');
 const connectUtility = require('../utility/connectUtility');
 
-async function queryReader({ connection, sql }) {
-  console.log(`Reading query ${sql}`);
+async function queryReader({
+  connection,
+  query,
+  queryType,
+  // obsolete; use query instead
+  sql,
+}) {
+  // if (sql && json) {
+  //   throw new Error('Only one of sql or json could be set');
+  // }
+  // if (!sql && !json) {
+  //   throw new Error('One of sql or json must be set');
+  // }
+  console.log(`Reading query ${query || sql}`);
+  // else console.log(`Reading query ${JSON.stringify(json)}`);
 
   const driver = requireEngineDriver(connection);
-  const pool = await connectUtility(driver, connection);
+  const pool = await connectUtility(driver, connection, queryType == 'json' ? 'read' : 'script');
   console.log(`Connected.`);
-  return await driver.readQuery(pool, sql);
+  return queryType == 'json' ? await driver.readJsonQuery(pool, query) : await driver.readQuery(pool, query || sql);
 }
 
 module.exports = queryReader;

packages/api/src/shell/requirePlugin.js

@@ -1,6 +1,8 @@
 const path = require('path');
-const { pluginsdir } = require('../utility/directories');
+const fs = require('fs');
+const { pluginsdir, packagedPluginsDir, getPluginBackendPath } = require('../utility/directories');
 const nativeModules = require('../nativeModules');
+const platformInfo = require('../utility/platformInfo');
 
 const loadedPlugins = {};
 
@@ -8,20 +10,19 @@ const dbgateEnv = {
   dbgateApi: null,
   nativeModules,
 };
-
 function requirePlugin(packageName, requiredPlugin = null) {
   if (!packageName) throw new Error('Missing packageName in plugin');
   if (loadedPlugins[packageName]) return loadedPlugins[packageName];
 
   if (requiredPlugin == null) {
     let module;
-    const modulePath = path.join(pluginsdir(), packageName, 'dist', 'backend.js');
+    const modulePath = getPluginBackendPath(packageName);
     console.log(`Loading module ${packageName} from ${modulePath}`);
     try {
       // @ts-ignore
       module = __non_webpack_require__(modulePath);
     } catch (err) {
-      console.error('Failed load webpacked module', err);
+      // console.log('Failed load webpacked module', err.message);
       module = require(modulePath);
     }
     requiredPlugin = module.__esModule ? module.default : module;

packages/api/src/shell/runScript.js

@@ -1,7 +1,8 @@
 const childProcessChecker = require('../utility/childProcessChecker');
+const processArgs = require('../utility/processArgs');
 
 async function runScript(func) {
-  if (process.argv.includes('--checkParent')) {
+  if (processArgs.checkParent) {
     childProcessChecker();
   }
   try {

packages/api/src/shell/sqlDataWriter.js

@@ -0,0 +1,50 @@
+const fs = require('fs');
+const stream = require('stream');
+const path = require('path');
+const { driverBase } = require('dbgate-tools');
+const requireEngineDriver = require('../utility/requireEngineDriver');
+
+class SqlizeStream extends stream.Transform {
+  constructor({ fileName }) {
+    super({ objectMode: true });
+    this.wasHeader = false;
+    this.tableName = path.parse(fileName).name;
+    this.driver = driverBase;
+  }
+  _transform(chunk, encoding, done) {
+    let skip = false;
+    if (!this.wasHeader) {
+      if (chunk.__isStreamHeader) {
+        skip = true;
+        this.tableName = chunk.pureName;
+        if (chunk.engine) {
+          // @ts-ignore
+          this.driver = requireEngineDriver(chunk.engine) || driverBase;
+        }
+      }
+      this.wasHeader = true;
+    }
+    if (!skip) {
+      const dmp = this.driver.createDumper();
+      dmp.put(
+        '^insert ^into %f (%,i) ^values (%,v);\n',
+        { pureName: this.tableName },
+        Object.keys(chunk),
+        Object.values(chunk)
+      );
+      this.push(dmp.s);
+    }
+    done();
+  }
+}
+
+async function sqlDataWriter({ fileName, driver, encoding = 'utf-8' }) {
+  console.log(`Writing file ${fileName}`);
+  const stringify = new SqlizeStream({ fileName });
+  const fileStream = fs.createWriteStream(fileName, encoding);
+  stringify.pipe(fileStream);
+  stringify['finisher'] = fileStream;
+  return stringify;
+}
+
+module.exports = sqlDataWriter;

packages/api/src/shell/tableReader.js

@@ -1,15 +1,21 @@
 const { quoteFullName, fullNameToString } = require('dbgate-tools');
 const requireEngineDriver = require('../utility/requireEngineDriver');
-const { decryptConnection } = require('../utility/crypting');
 const connectUtility = require('../utility/connectUtility');
 
 async function tableReader({ connection, pureName, schemaName }) {
   const driver = requireEngineDriver(connection);
-  const pool = await connectUtility(driver, connection);
+  const pool = await connectUtility(driver, connection, 'read');
   console.log(`Connected.`);
 
   const fullName = { pureName, schemaName };
 
+  if (driver.databaseEngineTypes.includes('document')) {
+    // @ts-ignore
+    console.log(`Reading collection ${fullNameToString(fullName)}`);
+    // @ts-ignore
+    return await driver.readQuery(pool, JSON.stringify(fullName));
+  }
+
   const table = await driver.analyseSingleObject(pool, fullName, 'tables');
   const query = `select * from ${quoteFullName(driver.dialect, fullName)}`;
   if (table) {

packages/api/src/shell/tableWriter.js

@@ -1,13 +1,15 @@
 const { fullNameToString } = require('dbgate-tools');
 const requireEngineDriver = require('../utility/requireEngineDriver');
-const { decryptConnection } = require('../utility/crypting');
 const connectUtility = require('../utility/connectUtility');
 
-async function tableWriter({ connection, schemaName, pureName, ...options }) {
+async function tableWriter({ connection, schemaName, pureName, driver, systemConnection, ...options }) {
   console.log(`Writing table ${fullNameToString({ schemaName, pureName })}`);
 
-  const driver = requireEngineDriver(connection);
-  const pool = await connectUtility(driver, connection);
+  if (!driver) {
+    driver = requireEngineDriver(connection);
+  }
+  const pool = systemConnection || (await connectUtility(driver, connection, 'write'));
+
   console.log(`Connected.`);
   return await driver.writeTable(pool, { schemaName, pureName }, options);
 }

packages/api/src/utility/ColumnMapTransformStream.js

@@ -0,0 +1,21 @@
+const stream = require('stream');
+const { transformRowUsingColumnMap } = require('dbgate-tools');
+
+class ColumnMapTransformStream extends stream.Transform {
+  constructor(columns) {
+    super({ objectMode: true });
+    this.columns = columns;
+  }
+  _transform(chunk, encoding, done) {
+    if (chunk.__isStreamHeader) {
+      // skip stream header
+      done();
+      return;
+    }
+
+    this.push(transformRowUsingColumnMap(chunk, this.columns));
+    done();
+  }
+}
+
+module.exports = ColumnMapTransformStream;

packages/api/src/utility/DatastoreProxy.js

@@ -1,6 +1,7 @@
 const { fork } = require('child_process');
 const uuidv1 = require('uuid/v1');
 const { handleProcessCommunication } = require('./processComm');
+const processArgs = require('../utility/processArgs');
 
 class DatastoreProxy {
   constructor(file) {
@@ -29,7 +30,13 @@ class DatastoreProxy {
 
   async ensureSubprocess() {
     if (!this.subprocess) {
-      this.subprocess = fork(process.argv[1], ['jslDatastoreProcess', ...process.argv.slice(3)]);
+      this.subprocess = fork(global['API_PACKAGE'] || process.argv[1], [
+        '--is-forked-api',
+        '--start-process',
+        'jslDatastoreProcess',
+        ...processArgs.getPassArgs(),
+        // ...process.argv.slice(3),
+      ]);
 
       this.subprocess.on('message', message => {
         // @ts-ignore

packages/api/src/utility/EnsureStreamHeaderStream.js

@@ -0,0 +1,31 @@
+const stream = require('stream');
+
+class EnsureStreamHeaderStream extends stream.Transform {
+  constructor() {
+    super({ objectMode: true });
+    this.wasHeader = false;
+  }
+  _transform(chunk, encoding, done) {
+    if (!this.wasHeader) {
+      if (chunk.__isDynamicStructure) {
+        // ignore dynamic structure header
+        done();
+        return;
+      }
+
+      if (!chunk.__isStreamHeader) {
+        this.push({
+          __isStreamHeader: true,
+          __isComputedStructure: true,
+          columns: Object.keys(chunk).map(columnName => ({ columnName })),
+        });
+      }
+
+      this.wasHeader = true;
+    }
+    this.push(chunk);
+    done();
+  }
+}
+
+module.exports = EnsureStreamHeaderStream;

packages/api/src/utility/JsonLinesDatabase.js

@@ -0,0 +1,142 @@
+const AsyncLock = require('async-lock');
+const fs = require('fs-extra');
+const uuidv1 = require('uuid/v1');
+
+const lock = new AsyncLock();
+
+// const lineReader = require('line-reader');
+// const { fetchNextLineFromReader } = require('./JsonLinesDatastore');
+
+class JsonLinesDatabase {
+  constructor(filename) {
+    this.filename = filename;
+    this.data = [];
+    this.loadedOk = false;
+    this.loadPerformed = false;
+  }
+
+  async _save() {
+    if (!this.loadedOk) {
+      // don't override data
+      return;
+    }
+    await fs.writeFile(this.filename, this.data.map(x => JSON.stringify(x)).join('\n'));
+  }
+
+  async _ensureLoaded() {
+    if (!this.loadPerformed) {
+      await lock.acquire('reader', async () => {
+        if (!this.loadPerformed) {
+          if (!(await fs.exists(this.filename))) {
+            this.loadedOk = true;
+            this.loadPerformed = true;
+            return;
+          }
+          try {
+            const text = await fs.readFile(this.filename, { encoding: 'utf-8' });
+            this.data = text
+              .split('\n')
+              .filter(x => x.trim())
+              .map(x => JSON.parse(x));
+            this.loadedOk = true;
+          } catch (err) {
+            console.error(`Error loading file ${this.filename}`, err);
+          }
+          this.loadPerformed = true;
+        }
+      });
+    }
+  }
+
+  async insert(obj) {
+    await this._ensureLoaded();
+    if (obj._id && (await this.get(obj._id))) {
+      throw new Error(`Cannot insert duplicate ID ${obj._id} into ${this.filename}`);
+    }
+    const elem = obj._id
+      ? obj
+      : {
+          ...obj,
+          _id: uuidv1(),
+        };
+    this.data.push(elem);
+    await this._save();
+    return elem;
+  }
+
+  async get(id) {
+    await this._ensureLoaded();
+    return this.data.find(x => x._id == id);
+  }
+
+  async find(cond) {
+    await this._ensureLoaded();
+    if (cond) {
+      return this.data.filter(x => {
+        for (const key of Object.keys(cond)) {
+          if (x[key] != cond[key]) return false;
+        }
+        return true;
+      });
+    } else {
+      return this.data;
+    }
+  }
+
+  async update(obj) {
+    await this._ensureLoaded();
+    this.data = this.data.map(x => (x._id == obj._id ? obj : x));
+    await this._save();
+    return obj;
+  }
+
+  async patch(id, values) {
+    await this._ensureLoaded();
+    this.data = this.data.map(x => (x._id == id ? { ...x, ...values } : x));
+    await this._save();
+    return this.data.find(x => x._id == id);
+  }
+
+  async remove(id) {
+    await this._ensureLoaded();
+    const removed = this.data.find(x => x._id == id);
+    this.data = this.data.filter(x => x._id != id);
+    await this._save();
+    return removed;
+  }
+
+  //   async _openReader() {
+  //     return new Promise((resolve, reject) =>
+  //       lineReader.open(this.filename, (err, reader) => {
+  //         if (err) reject(err);
+  //         resolve(reader);
+  //       })
+  //     );
+  //   }
+
+  //   async _read() {
+  //     this.data = [];
+  //     if (!(await fs.exists(this.filename))) return;
+  //     try {
+  //       const reader = await this._openReader();
+  //       for (;;) {
+  //         const line = await fetchNextLineFromReader(reader);
+  //         if (!line) break;
+  //         this.data.push(JSON.parse(line));
+  //       }
+  //     } catch (err) {
+  //       console.error(`Error loading file ${this.filename}`, err);
+  //     }
+  //   }
+
+  //   async _write() {
+  //     const fw = fs.createWriteStream(this.filename);
+  //     for (const obj of this.data) {
+  //       await fw.write(JSON.stringify(obj));
+  //       await fw.write('\n');
+  //     }
+  //     await fw.end();
+  //   }
+}
+
+module.exports = JsonLinesDatabase;