raw data test

feat: filter extentions on fe

.github/workflows/build-app-beta.yaml

@@ -1,83 +1,86 @@
+# --------------------------------------------------------------------------------------------
+# This file is generated. Do not edit manually
+# --------------------------------------------------------------------------------------------
 name: Electron app BETA
-
-on:
+'on':
   push:
     tags:
       - 'v[0-9]+.[0-9]+.[0-9]+-beta.[0-9]+'
-
 jobs:
   build:
-    runs-on: ${{ matrix.os }}
-
+    runs-on: '${{ matrix.os }}'
     strategy:
       fail-fast: false
       matrix:
-        os: [macos-12, windows-2022, ubuntu-22.04]
-        # os: [macOS-10.15]
-
+        os:
+          - macos-14
+          - windows-2022
+          - ubuntu-22.04
     steps:
+      - name: Install python 3.11 (MacOS)
+        if: matrix.os == 'macos-14'
+        run: |
+          brew install python@3.11
+          echo "PYTHON=/opt/homebrew/bin/python3.11" >> $GITHUB_ENV
       - name: Context
         env:
-          GITHUB_CONTEXT: ${{ toJson(github) }}
+          GITHUB_CONTEXT: '${{ toJson(github) }}'
         run: echo "$GITHUB_CONTEXT"
       - uses: actions/checkout@v2
         with:
           fetch-depth: 1
-      - name: Use Node.js 18.x
+      - name: Use Node.js 22.x
         uses: actions/setup-node@v1
         with:
-          node-version: 18.x
-      - name: yarn adjustPackageJson
+          node-version: 22.x
+      - name: adjustPackageJson
         run: |
-          yarn adjustPackageJson
+
+          node adjustPackageJson --community
       - name: setUpdaterChannel beta
         run: |
+
           node setUpdaterChannel beta
       - name: yarn set timeout
         run: |
+
           yarn config set network-timeout 100000
       - name: yarn install
         run: |
+
           yarn install
       - name: setCurrentVersion
         run: |
+
           yarn setCurrentVersion
       - name: printSecrets
         run: |
-            yarn printSecrets
+
+          yarn printSecrets
         env:
-          GIST_UPLOAD_SECRET : ${{secrets.GIST_UPLOAD_SECRET}}
+          GIST_UPLOAD_SECRET: '${{secrets.GIST_UPLOAD_SECRET}}'
       - name: fillPackagedPlugins
         run: |
+
           yarn fillPackagedPlugins
       - name: Install Snapcraft
         if: matrix.os == 'ubuntu-22.04'
         uses: samuelmeuli/action-snapcraft@v1
       - name: Publish
         run: |
+
           yarn run build:app
         env:
-          GH_TOKEN: ${{ secrets.GH_TOKEN }} # token for electron publish
-
-          WIN_CSC_LINK: ${{ secrets.WINCERT_2025 }}
-          WIN_CSC_KEY_PASSWORD: ${{ secrets.WINCERT_2025_PASSWORD }}
-          # WIN_CSC_LINK: ${{ secrets.WINCERT_CERTIFICATE }}
-          # WIN_CSC_KEY_PASSWORD: ${{ secrets.WINCERT_PASSWORD }}
-
-          CSC_LINK: ${{ secrets.APPLECERT_CERTIFICATE }}
-          CSC_KEY_PASSWORD: ${{ secrets.APPLECERT_PASSWORD }}
-
-          APPLE_ID: ${{ secrets.APPLE_ID }}
-          APPLE_TEAM_ID: ${{ secrets.APPLE_TEAM_ID }}
-          APPLE_ID_PASSWORD: ${{ secrets.APPLE_ID_PASSWORD }}
-
-      - name: publishSnap
-        if: matrix.os == 'ubuntu-22.04'
-        run: |
-          snapcraft upload --release=beta app/dist/*.snap
-        env:
-          SNAPCRAFT_STORE_CREDENTIALS: ${{secrets.SNAPCRAFT_LOGIN}}
-
+          GH_TOKEN: '${{ secrets.GH_TOKEN }}'
+          WIN_CSC_LINK: '${{ secrets.WINCERT_2025 }}'
+          WIN_CSC_KEY_PASSWORD: '${{ secrets.WINCERT_2025_PASSWORD }}'
+          CSC_LINK: '${{ secrets.APPLECERT_CERTIFICATE }}'
+          CSC_KEY_PASSWORD: '${{ secrets.APPLECERT_PASSWORD }}'
+          APPLE_ID: '${{ secrets.APPLE_ID }}'
+          APPLE_TEAM_ID: '${{ secrets.APPLE_TEAM_ID }}'
+          APPLE_ID_PASSWORD: '${{ secrets.APPLE_ID_PASSWORD }}'
+          SNAPCRAFT_STORE_CREDENTIALS: '${{secrets.SNAPCRAFT_LOGIN}}'
+          APPLE_APP_SPECIFIC_PASSWORD: '${{secrets.APPLE_APP_SPECIFIC_PASSWORD}}'
       - name: Copy artifacts
         run: |
           mkdir artifacts          
@@ -89,8 +92,10 @@ jobs:
           cp app/dist/*win*.exe artifacts/dbgate-beta.exe || true
           cp app/dist/*win_x64.zip artifacts/dbgate-windows-beta.zip || true
           cp app/dist/*win_arm64.zip artifacts/dbgate-windows-beta-arm64.zip || true
-          cp app/dist/*-mac_x64.dmg artifacts/dbgate-beta.dmg || true
+          cp app/dist/*-mac_universal.dmg artifacts/dbgate-beta.dmg || true
+          cp app/dist/*-mac_x64.dmg artifacts/dbgate-beta-x64.dmg || true
           cp app/dist/*-mac_arm64.dmg artifacts/dbgate-beta-arm64.dmg || true
+          mv app/dist/*.snap artifacts/dbgate-beta.snap || true
 
           mv app/dist/*.exe artifacts/ || true
           mv app/dist/*.zip artifacts/ || true
@@ -103,18 +108,21 @@ jobs:
 
           mv app/dist/*.yml artifacts/ || true
           rm artifacts/builder-debug.yml
-
       - name: Upload artifacts
         uses: actions/upload-artifact@v4
         with:
-          name: ${{ matrix.os }}
+          name: '${{ matrix.os }}'
           path: artifacts
-
       - name: Release
         uses: softprops/action-gh-release@v1
-        if: startsWith(github.ref, 'refs/tags/')
+        if: 'startsWith(github.ref, ''refs/tags/'')'
         with:
-          files: 'artifacts/**'
+          files: artifacts/**
           prerelease: true
         env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GITHUB_TOKEN: '${{ secrets.GITHUB_TOKEN }}'
+      - name: Print content of notarization-error.log
+        if: failure() && matrix.os == 'macos-14'
+        run: |
+
+          find . -type f -name "notarization-error.log" -exec echo "=== Start of {} ===" \; -exec cat {} \; -exec echo "=== End of {} ===" \;

.github/workflows/build-app-pro-beta.yaml

@@ -1,43 +1,44 @@
+# --------------------------------------------------------------------------------------------
+# This file is generated. Do not edit manually
+# --------------------------------------------------------------------------------------------
 name: Electron app PREMIUM BETA
-
-on:
+'on':
   push:
     tags:
       - 'v[0-9]+.[0-9]+.[0-9]+-premium-beta.[0-9]+'
-
 jobs:
   build:
-    runs-on: ${{ matrix.os }}
-
+    runs-on: '${{ matrix.os }}'
     strategy:
       fail-fast: false
       matrix:
-        # os: [windows-2022]
-        # os: [ubuntu-22.04]
-        # os: [windows-2022, ubuntu-22.04]
-        os: [macos-12, windows-2022, ubuntu-22.04]
-        # os: [macOS-10.15]
-
+        os:
+          - macos-14
+          - windows-2022
+          - ubuntu-22.04
     steps:
+      - name: Install python 3.11 (MacOS)
+        if: matrix.os == 'macos-14'
+        run: |
+          brew install python@3.11
+          echo "PYTHON=/opt/homebrew/bin/python3.11" >> $GITHUB_ENV
       - name: Context
         env:
-          GITHUB_CONTEXT: ${{ toJson(github) }}
+          GITHUB_CONTEXT: '${{ toJson(github) }}'
         run: echo "$GITHUB_CONTEXT"
       - uses: actions/checkout@v2
         with:
           fetch-depth: 1
-      - name: Use Node.js 18.x
+      - name: Use Node.js 22.x
         uses: actions/setup-node@v1
         with:
-          node-version: 18.x
-          
+          node-version: 22.x
       - name: Checkout dbgate/dbgate-pro
         uses: actions/checkout@v2
         with:
           repository: dbgate/dbgate-pro
-          token: ${{ secrets.GH_TOKEN }}
+          token: '${{ secrets.GH_TOKEN }}'
           path: dbgate-pro
-
       - name: Merge dbgate/dbgate-pro
         run: |
           mkdir ../dbgate-pro
@@ -49,76 +50,82 @@ jobs:
           yarn
           node sync.js --nowatch
           cd ..
+      - name: adjustPackageJson
+        run: |
+          cd ..
+          cd dbgate-merged
 
-      - name: yarn adjustPackageJson
-        run: |
-          cd ..
-          cd dbgate-merged
-          yarn adjustPackageJson
-      - name: adjustAppPackageJsonPremium
-        run: |
-          cd ..
-          cd dbgate-merged
-          node adjustAppPackageJsonPremium
+          node adjustPackageJson --premium
       - name: setUpdaterChannel premium-beta
         run: |
           cd ..
           cd dbgate-merged
+
           node setUpdaterChannel premium-beta
       - name: yarn set timeout
         run: |
           cd ..
           cd dbgate-merged
+
           yarn config set network-timeout 100000
       - name: yarn install
         run: |
           cd ..
           cd dbgate-merged
+
           yarn install
       - name: setCurrentVersion
         run: |
           cd ..
           cd dbgate-merged
+
           yarn setCurrentVersion
       - name: printSecrets
         run: |
           cd ..
           cd dbgate-merged
-            yarn printSecrets
+
+          yarn printSecrets
         env:
-          GIST_UPLOAD_SECRET : ${{secrets.GIST_UPLOAD_SECRET}}
+          GIST_UPLOAD_SECRET: '${{secrets.GIST_UPLOAD_SECRET}}'
       - name: fillPackagedPlugins
         run: |
           cd ..
           cd dbgate-merged
+
           yarn fillPackagedPlugins
       - name: Publish
         run: |
           cd ..
           cd dbgate-merged
+
           yarn run build:app
         env:
-          GH_TOKEN: ${{ secrets.GH_TOKEN }} # token for electron publish
-
-          WIN_CSC_LINK: ${{ secrets.WINCERT_2025 }}
-          WIN_CSC_KEY_PASSWORD: ${{ secrets.WINCERT_2025_PASSWORD }}
-          # WIN_CSC_LINK: ${{ secrets.WINCERT_CERTIFICATE }}
-          # WIN_CSC_KEY_PASSWORD: ${{ secrets.WINCERT_PASSWORD }}
-
-          CSC_LINK: ${{ secrets.APPLECERT_CERTIFICATE }}
-          CSC_KEY_PASSWORD: ${{ secrets.APPLECERT_PASSWORD }}
-
-          APPLE_ID: ${{ secrets.APPLE_ID }}
-          APPLE_TEAM_ID: ${{ secrets.APPLE_TEAM_ID }}
-          APPLE_ID_PASSWORD: ${{ secrets.APPLE_ID_PASSWORD }}
-
+          GH_TOKEN: '${{ secrets.GH_TOKEN }}'
+          WIN_CSC_LINK: '${{ secrets.WINCERT_2025 }}'
+          WIN_CSC_KEY_PASSWORD: '${{ secrets.WINCERT_2025_PASSWORD }}'
+          CSC_LINK: '${{ secrets.APPLECERT_CERTIFICATE }}'
+          CSC_KEY_PASSWORD: '${{ secrets.APPLECERT_PASSWORD }}'
+          APPLE_ID: '${{ secrets.APPLE_ID }}'
+          APPLE_TEAM_ID: '${{ secrets.APPLE_TEAM_ID }}'
+          APPLE_ID_PASSWORD: '${{ secrets.APPLE_ID_PASSWORD }}'
+          SNAPCRAFT_STORE_CREDENTIALS: '${{secrets.SNAPCRAFT_LOGIN}}'
+          APPLE_APP_SPECIFIC_PASSWORD: '${{secrets.APPLE_APP_SPECIFIC_PASSWORD}}'
       - name: Copy artifacts
         run: |
           mkdir artifacts          
 
+          cp ../dbgate-merged/app/dist/*.deb artifacts/dbgate-premium-beta.deb || true
           cp ../dbgate-merged/app/dist/*x86*.AppImage artifacts/dbgate-premium-beta.AppImage || true
+          cp ../dbgate-merged/app/dist/*arm64*.AppImage artifacts/dbgate-premium-beta-arm64.AppImage || true
+          cp ../dbgate-merged/app/dist/*armv7l*.AppImage artifacts/dbgate-premium-beta-armv7l.AppImage || true
           cp ../dbgate-merged/app/dist/*win*.exe artifacts/dbgate-premium-beta.exe || true
-          cp ../dbgate-merged/app/dist/*-mac_x64.dmg artifacts/dbgate-premium-beta.dmg || true
+          cp ../dbgate-merged/app/dist/*win_x64.zip artifacts/dbgate-windows-premium-beta.zip || true
+          cp ../dbgate-merged/app/dist/*win_arm64.zip artifacts/dbgate-windows-premium-beta-arm64.zip || true
+          cp ../dbgate-merged/app/dist/*-mac_universal.dmg artifacts/dbgate-premium-beta.dmg || true
+          cp ../dbgate-merged/app/dist/*-mac_x64.dmg artifacts/dbgate-premium-beta-x64.dmg || true
+          cp ../dbgate-merged/app/dist/*-mac_arm64.dmg artifacts/dbgate-premium-beta-arm64.dmg || true
+          mv ../dbgate-merged/app/dist/*.snap artifacts/dbgate-premium-beta.snap || true
 
           mv ../dbgate-merged/app/dist/*.exe artifacts/ || true
           mv ../dbgate-merged/app/dist/*.zip artifacts/ || true
@@ -131,18 +138,23 @@ jobs:
 
           mv ../dbgate-merged/app/dist/*.yml artifacts/ || true
           rm artifacts/builder-debug.yml
-
       - name: Upload artifacts
         uses: actions/upload-artifact@v4
         with:
-          name: ${{ matrix.os }}
+          name: '${{ matrix.os }}'
           path: artifacts
-
       - name: Release
         uses: softprops/action-gh-release@v1
-        if: startsWith(github.ref, 'refs/tags/')
+        if: 'startsWith(github.ref, ''refs/tags/'')'
         with:
-          files: 'artifacts/**'
+          files: artifacts/**
           prerelease: true
         env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GITHUB_TOKEN: '${{ secrets.GITHUB_TOKEN }}'
+      - name: Print content of notarization-error.log
+        if: failure() && matrix.os == 'macos-14'
+        run: |
+          cd ..
+          cd dbgate-merged
+
+          find . -type f -name "notarization-error.log" -exec echo "=== Start of {} ===" \; -exec cat {} \; -exec echo "=== End of {} ===" \;

.github/workflows/build-app-pro.yaml

@@ -1,44 +1,44 @@
+# --------------------------------------------------------------------------------------------
+# This file is generated. Do not edit manually
+# --------------------------------------------------------------------------------------------
 name: Electron app PREMIUM
-
-on:
+'on':
   push:
     tags:
       - 'v[0-9]+.[0-9]+.[0-9]+'
-      # - 'v*' # Push events to matching v*, i.e. v1.0, v20.15.10
-
-    # branches:
-    #   - production
-
 jobs:
   build:
-    runs-on: ${{ matrix.os }}
-
+    runs-on: '${{ matrix.os }}'
     strategy:
       fail-fast: false
       matrix:
-        # os: [ubuntu-22.04, windows-2016]
-        os: [macos-12, windows-2022, ubuntu-22.04]
-
+        os:
+          - macos-14
+          - windows-2022
+          - ubuntu-22.04
     steps:
+      - name: Install python 3.11 (MacOS)
+        if: matrix.os == 'macos-14'
+        run: |
+          brew install python@3.11
+          echo "PYTHON=/opt/homebrew/bin/python3.11" >> $GITHUB_ENV
       - name: Context
         env:
-          GITHUB_CONTEXT: ${{ toJson(github) }}
+          GITHUB_CONTEXT: '${{ toJson(github) }}'
         run: echo "$GITHUB_CONTEXT"
       - uses: actions/checkout@v2
         with:
           fetch-depth: 1
-      - name: Use Node.js 18.x
+      - name: Use Node.js 22.x
         uses: actions/setup-node@v1
         with:
-          node-version: 18.x
-
+          node-version: 22.x
       - name: Checkout dbgate/dbgate-pro
         uses: actions/checkout@v2
         with:
           repository: dbgate/dbgate-pro
-          token: ${{ secrets.GH_TOKEN }}
+          token: '${{ secrets.GH_TOKEN }}'
           path: dbgate-pro
-
       - name: Merge dbgate/dbgate-pro
         run: |
           mkdir ../dbgate-pro
@@ -50,101 +50,111 @@ jobs:
           yarn
           node sync.js --nowatch
           cd ..
+      - name: adjustPackageJson
+        run: |
+          cd ..
+          cd dbgate-merged
 
-      - name: yarn adjustPackageJson
-        run: |
-          cd ..
-          cd dbgate-merged
-          yarn adjustPackageJson
-      - name: adjustAppPackageJsonPremium
-        run: |
-          cd ..
-          cd dbgate-merged
-          node adjustAppPackageJsonPremium
+          node adjustPackageJson --premium
       - name: setUpdaterChannel premium
         run: |
           cd ..
           cd dbgate-merged
+
           node setUpdaterChannel premium
       - name: yarn set timeout
         run: |
           cd ..
           cd dbgate-merged
+
           yarn config set network-timeout 100000
       - name: yarn install
         run: |
           cd ..
           cd dbgate-merged
+
           yarn install
       - name: setCurrentVersion
         run: |
           cd ..
           cd dbgate-merged
+
           yarn setCurrentVersion
       - name: printSecrets
         run: |
           cd ..
           cd dbgate-merged
+
           yarn printSecrets
         env:
-          GIST_UPLOAD_SECRET : ${{secrets.GIST_UPLOAD_SECRET}}
+          GIST_UPLOAD_SECRET: '${{secrets.GIST_UPLOAD_SECRET}}'
       - name: fillPackagedPlugins
         run: |
           cd ..
           cd dbgate-merged
+
           yarn fillPackagedPlugins
       - name: Publish
         run: |
           cd ..
           cd dbgate-merged
+
           yarn run build:app
         env:
-          GH_TOKEN: ${{ secrets.GH_TOKEN }} # token for electron publish
-
-          WIN_CSC_LINK: ${{ secrets.WINCERT_2025 }}
-          WIN_CSC_KEY_PASSWORD: ${{ secrets.WINCERT_2025_PASSWORD }}
-          # WIN_CSC_LINK: ${{ secrets.WINCERT_CERTIFICATE }}
-          # WIN_CSC_KEY_PASSWORD: ${{ secrets.WINCERT_PASSWORD }}
-
-          CSC_LINK: ${{ secrets.APPLECERT_CERTIFICATE }}
-          CSC_KEY_PASSWORD: ${{ secrets.APPLECERT_PASSWORD }}
-
-          APPLE_ID: ${{ secrets.APPLE_ID }}
-          APPLE_TEAM_ID: ${{ secrets.APPLE_TEAM_ID }}
-          APPLE_ID_PASSWORD: ${{ secrets.APPLE_ID_PASSWORD }}
-
+          GH_TOKEN: '${{ secrets.GH_TOKEN }}'
+          WIN_CSC_LINK: '${{ secrets.WINCERT_2025 }}'
+          WIN_CSC_KEY_PASSWORD: '${{ secrets.WINCERT_2025_PASSWORD }}'
+          CSC_LINK: '${{ secrets.APPLECERT_CERTIFICATE }}'
+          CSC_KEY_PASSWORD: '${{ secrets.APPLECERT_PASSWORD }}'
+          APPLE_ID: '${{ secrets.APPLE_ID }}'
+          APPLE_TEAM_ID: '${{ secrets.APPLE_TEAM_ID }}'
+          APPLE_ID_PASSWORD: '${{ secrets.APPLE_ID_PASSWORD }}'
+          SNAPCRAFT_STORE_CREDENTIALS: '${{secrets.SNAPCRAFT_LOGIN}}'
+          APPLE_APP_SPECIFIC_PASSWORD: '${{secrets.APPLE_APP_SPECIFIC_PASSWORD}}'
       - name: Copy artifacts
         run: |
           mkdir artifacts          
 
+          cp ../dbgate-merged/app/dist/*.deb artifacts/dbgate-premium-latest.deb || true
           cp ../dbgate-merged/app/dist/*x86*.AppImage artifacts/dbgate-premium-latest.AppImage || true
-          cp ../dbgate-merged/app/dist/*.exe artifacts/dbgate-premium-latest.exe || true
-          cp ../dbgate-merged/app/dist/*win_x64.zip artifacts/dbgate-premium-windows-latest.zip || true
-          cp ../dbgate-merged/app/dist/*win_arm64.zip artifacts/dbgate-premium-windows-latest-arm64.zip || true
+          cp ../dbgate-merged/app/dist/*arm64*.AppImage artifacts/dbgate-premium-latest-arm64.AppImage || true
+          cp ../dbgate-merged/app/dist/*armv7l*.AppImage artifacts/dbgate-premium-latest-armv7l.AppImage || true
+          cp ../dbgate-merged/app/dist/*win*.exe artifacts/dbgate-premium-latest.exe || true
+          cp ../dbgate-merged/app/dist/*win_x64.zip artifacts/dbgate-windows-premium-latest.zip || true
+          cp ../dbgate-merged/app/dist/*win_arm64.zip artifacts/dbgate-windows-premium-latest-arm64.zip || true
           cp ../dbgate-merged/app/dist/*-mac_universal.dmg artifacts/dbgate-premium-latest.dmg || true
           cp ../dbgate-merged/app/dist/*-mac_x64.dmg artifacts/dbgate-premium-latest-x64.dmg || true
+          cp ../dbgate-merged/app/dist/*-mac_arm64.dmg artifacts/dbgate-premium-latest-arm64.dmg || true
+          mv ../dbgate-merged/app/dist/*.snap artifacts/dbgate-premium-latest.snap || true
 
           mv ../dbgate-merged/app/dist/*.exe artifacts/ || true
           mv ../dbgate-merged/app/dist/*.zip artifacts/ || true
           mv ../dbgate-merged/app/dist/*.tar.gz artifacts/ || true
           mv ../dbgate-merged/app/dist/*.AppImage artifacts/ || true
           mv ../dbgate-merged/app/dist/*.deb artifacts/ || true
+          mv ../dbgate-merged/app/dist/*.snap artifacts/ || true
           mv ../dbgate-merged/app/dist/*.dmg artifacts/ || true
           mv ../dbgate-merged/app/dist/*.blockmap artifacts/ || true
 
           mv ../dbgate-merged/app/dist/*.yml artifacts/ || true
           rm artifacts/builder-debug.yml
-
       - name: Upload artifacts
         uses: actions/upload-artifact@v4
         with:
-          name: ${{ matrix.os }}
+          name: '${{ matrix.os }}'
           path: artifacts
-
       - name: Release
         uses: softprops/action-gh-release@v1
-        if: startsWith(github.ref, 'refs/tags/')
+        if: 'startsWith(github.ref, ''refs/tags/'')'
         with:
-          files: 'artifacts/**'
+          files: artifacts/**
+          prerelease: false
         env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GITHUB_TOKEN: '${{ secrets.GITHUB_TOKEN }}'
+      - name: Print content of notarization-error.log
+        if: failure() && matrix.os == 'macos-14'
+        run: |
+          cd ..
+          cd dbgate-merged
+
+          find . -type f -name "notarization-error.log" -exec echo "=== Start of {} ===" \; -exec cat {} \; -exec echo "=== End of {} ===" \;

.github/workflows/build-app.yaml

@@ -1,90 +1,85 @@
+# --------------------------------------------------------------------------------------------
+# This file is generated. Do not edit manually
+# --------------------------------------------------------------------------------------------
 name: Electron app
-
-on:
+'on':
   push:
     tags:
       - 'v[0-9]+.[0-9]+.[0-9]+'
-      # - 'v*' # Push events to matching v*, i.e. v1.0, v20.15.10
-
-    # branches:
-    #   - production
-
 jobs:
   build:
-    runs-on: ${{ matrix.os }}
-
+    runs-on: '${{ matrix.os }}'
     strategy:
       fail-fast: false
       matrix:
-        # os: [ubuntu-22.04, windows-2016]
-        os: [macos-12, windows-2022, ubuntu-22.04]
-
+        os:
+          - macos-14
+          - windows-2022
+          - ubuntu-22.04
     steps:
+      - name: Install python 3.11 (MacOS)
+        if: matrix.os == 'macos-14'
+        run: |
+          brew install python@3.11
+          echo "PYTHON=/opt/homebrew/bin/python3.11" >> $GITHUB_ENV
       - name: Context
         env:
-          GITHUB_CONTEXT: ${{ toJson(github) }}
+          GITHUB_CONTEXT: '${{ toJson(github) }}'
         run: echo "$GITHUB_CONTEXT"
       - uses: actions/checkout@v2
         with:
           fetch-depth: 1
-      - name: Use Node.js 18.x
+      - name: Use Node.js 22.x
         uses: actions/setup-node@v1
         with:
-          node-version: 18.x
-      - name: yarn adjustPackageJson
+          node-version: 22.x
+      - name: adjustPackageJson
         run: |
-          yarn adjustPackageJson
+
+          node adjustPackageJson --community
       - name: yarn set timeout
         run: |
+
           yarn config set network-timeout 100000
       - name: yarn install
         run: |
-          # yarn --version
-          # yarn config set network-timeout 300000
+
           yarn install
       - name: setCurrentVersion
         run: |
+
           yarn setCurrentVersion
       - name: printSecrets
         run: |
-            yarn printSecrets
+
+          yarn printSecrets
         env:
-          GIST_UPLOAD_SECRET : ${{secrets.GIST_UPLOAD_SECRET}}
+          GIST_UPLOAD_SECRET: '${{secrets.GIST_UPLOAD_SECRET}}'
       - name: fillPackagedPlugins
         run: |
+
           yarn fillPackagedPlugins
       - name: Install Snapcraft
         if: matrix.os == 'ubuntu-22.04'
         uses: samuelmeuli/action-snapcraft@v1
       - name: Publish
         run: |
+
           yarn run build:app
         env:
-          GH_TOKEN: ${{ secrets.GH_TOKEN }} # token for electron publish
-
-          WIN_CSC_LINK: ${{ secrets.WINCERT_2025 }}
-          WIN_CSC_KEY_PASSWORD: ${{ secrets.WINCERT_2025_PASSWORD }}
-          # WIN_CSC_LINK: ${{ secrets.WINCERT_CERTIFICATE }}
-          # WIN_CSC_KEY_PASSWORD: ${{ secrets.WINCERT_PASSWORD }}
-
-          CSC_LINK: ${{ secrets.APPLECERT_CERTIFICATE }}
-          CSC_KEY_PASSWORD: ${{ secrets.APPLECERT_PASSWORD }}
-
-          APPLE_ID: ${{ secrets.APPLE_ID }}
-          APPLE_TEAM_ID: ${{ secrets.APPLE_TEAM_ID }}
-          APPLE_ID_PASSWORD: ${{ secrets.APPLE_ID_PASSWORD }}
-
+          GH_TOKEN: '${{ secrets.GH_TOKEN }}'
+          WIN_CSC_LINK: '${{ secrets.WINCERT_2025 }}'
+          WIN_CSC_KEY_PASSWORD: '${{ secrets.WINCERT_2025_PASSWORD }}'
+          CSC_LINK: '${{ secrets.APPLECERT_CERTIFICATE }}'
+          CSC_KEY_PASSWORD: '${{ secrets.APPLECERT_PASSWORD }}'
+          APPLE_ID: '${{ secrets.APPLE_ID }}'
+          APPLE_TEAM_ID: '${{ secrets.APPLE_TEAM_ID }}'
+          APPLE_ID_PASSWORD: '${{ secrets.APPLE_ID_PASSWORD }}'
+          SNAPCRAFT_STORE_CREDENTIALS: '${{secrets.SNAPCRAFT_LOGIN}}'
+          APPLE_APP_SPECIFIC_PASSWORD: '${{secrets.APPLE_APP_SPECIFIC_PASSWORD}}'
       - name: generatePadFile
         run: |
           yarn generatePadFile
-
-      - name: publishSnap
-        if: matrix.os == 'ubuntu-22.04'
-        run: |
-          snapcraft upload --release=stable app/dist/*.snap
-        env:
-          SNAPCRAFT_STORE_CREDENTIALS: ${{secrets.SNAPCRAFT_LOGIN}}
-
       - name: Copy artifacts
         run: |
           mkdir artifacts          
@@ -93,74 +88,44 @@ jobs:
           cp app/dist/*x86*.AppImage artifacts/dbgate-latest.AppImage || true
           cp app/dist/*arm64*.AppImage artifacts/dbgate-latest-arm64.AppImage || true
           cp app/dist/*armv7l*.AppImage artifacts/dbgate-latest-armv7l.AppImage || true
-          cp app/dist/*.exe artifacts/dbgate-latest.exe || true
+          cp app/dist/*win*.exe artifacts/dbgate-latest.exe || true
           cp app/dist/*win_x64.zip artifacts/dbgate-windows-latest.zip || true
           cp app/dist/*win_arm64.zip artifacts/dbgate-windows-latest-arm64.zip || true
           cp app/dist/*-mac_universal.dmg artifacts/dbgate-latest.dmg || true
           cp app/dist/*-mac_x64.dmg artifacts/dbgate-latest-x64.dmg || true
+          cp app/dist/*-mac_arm64.dmg artifacts/dbgate-latest-arm64.dmg || true
+          mv app/dist/*.snap artifacts/dbgate-latest.snap || true
 
           mv app/dist/*.exe artifacts/ || true
           mv app/dist/*.zip artifacts/ || true
           mv app/dist/*.tar.gz artifacts/ || true
           mv app/dist/*.AppImage artifacts/ || true
           mv app/dist/*.deb artifacts/ || true
+          mv app/dist/*.snap artifacts/ || true
           mv app/dist/*.dmg artifacts/ || true
-          mv app/dist/*.snap artifacts/dbgate-latest.snap || true
           mv app/dist/*.blockmap artifacts/ || true
 
           mv app/dist/*.yml artifacts/ || true
           rm artifacts/builder-debug.yml
-
-      # - name: Copy artifacts Linux, MacOs
-      #   if: matrix.os != 'windows-2016'
-      #   run: |
-      #     mkdir artifacts
-
-      #     cp app/dist/*.AppImage artifacts/ || true
-      #     cp app/dist/*.dmg artifacts/ || true
-      #     cp app/dist/*.deb artifacts/ || true
-
-      #     mv app/dist/*.deb artifacts/dbgate-linux.deb || true
-      #     mv app/dist/*.AppImage artifacts/dbgate-linux.AppImage || true
-      #     mv app/dist/*.dmg artifacts/dbgate-mac.dmg || true
-
-      # - name: Copy artifacts Win
-      #   if: matrix.os == 'windows-2016'
-      #   run: |
-      #     mkdir artifacts
-
-      #     cp app/dist/*.exe artifacts/ || true
-
-      #     mv app/dist/*.exe artifacts/dbgate-windows.exe
-
-      #     mv app/dist/latest.yml artifacts/latest.yml || true
-
       - name: Copy PAD file
         if: matrix.os == 'windows-2022'
         run: |
           mv app/dist/dbgate-pad.xml artifacts/ || true
-
       - name: Upload artifacts
         uses: actions/upload-artifact@v4
         with:
-          name: ${{ matrix.os }}
+          name: '${{ matrix.os }}'
           path: artifacts
-
       - name: Release
         uses: softprops/action-gh-release@v1
-        if: startsWith(github.ref, 'refs/tags/')
+        if: 'startsWith(github.ref, ''refs/tags/'')'
         with:
-          files: 'artifacts/**'
+          files: artifacts/**
+          prerelease: false
         env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GITHUB_TOKEN: '${{ secrets.GITHUB_TOKEN }}'
+      - name: Print content of notarization-error.log
+        if: failure() && matrix.os == 'macos-14'
+        run: |
 
-      # - name: Create Release
-      #   id: create_release
-      #   uses: actions/create-release@v1
-      #   env:
-      #     GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      #   with:
-      #     tag_name: ${{ github.ref }}
-      #     release_name: Release ${{ github.ref }}
-      #     draft: false
-      #     prerelease: false
+          find . -type f -name "notarization-error.log" -exec echo "=== Start of {} ===" \; -exec cat {} \; -exec echo "=== End of {} ===" \;

.github/workflows/build-aws-pro-beta.yaml

@@ -1,47 +1,41 @@
+# --------------------------------------------------------------------------------------------
+# This file is generated. Do not edit manually
+# --------------------------------------------------------------------------------------------
 name: AWS image PREMIUM
-
-on:
+'on':
   push:
     tags:
       - 'v[0-9]+.[0-9]+.[0-9]+'
-      - 'v[0-9]+.[0-9]+.[0-9]+-premium-beta.[0-9]+'
       - 'v[0-9]+.[0-9]+.[0-9]+-packer-beta.[0-9]+'
-
 jobs:
   build:
-
-    runs-on: ${{ matrix.os }}
-
+    runs-on: '${{ matrix.os }}'
     strategy:
       matrix:
-        os: [ubuntu-22.04]
-
+        os:
+          - ubuntu-22.04
     steps:
       - name: Context
         env:
-          GITHUB_CONTEXT: ${{ toJson(github) }}
+          GITHUB_CONTEXT: '${{ toJson(github) }}'
         run: echo "$GITHUB_CONTEXT"
       - uses: actions/checkout@v2
         with:
           fetch-depth: 1
-
       - name: Use Node.js 18.x
         uses: actions/setup-node@v1
         with:
           node-version: 18.x
-
       - name: Setup `packer`
         uses: hashicorp/setup-packer@main
         with:
           version: latest
-
       - name: Checkout dbgate/dbgate-pro
         uses: actions/checkout@v2
         with:
           repository: dbgate/dbgate-pro
-          token: ${{ secrets.GH_TOKEN }}
+          token: '${{ secrets.GH_TOKEN }}'
           path: dbgate-pro
-
       - name: Merge dbgate/dbgate-pro
         run: |
           mkdir ../dbgate-pro
@@ -53,33 +47,28 @@ jobs:
           yarn
           node sync.js --nowatch
           cd ..
-
-      - name: yarn adjustPackageJson
+      - name: adjustPackageJson
         run: |
           cd ..
           cd dbgate-merged
-          yarn adjustPackageJson
-
+          node adjustPackageJson --premium
       - name: yarn install
         run: |
           cd ..
           cd dbgate-merged
           yarn install
-
       - name: setCurrentVersion
         run: |
           cd ..
           cd dbgate-merged
           yarn setCurrentVersion
-          
       - name: printSecrets
         run: |
           cd ..
           cd dbgate-merged
             yarn printSecrets
         env:
-          GIST_UPLOAD_SECRET : ${{secrets.GIST_UPLOAD_SECRET}}
-
+          GIST_UPLOAD_SECRET: '${{secrets.GIST_UPLOAD_SECRET}}'
       - name: Prepare packer build
         run: |
           cd ..
@@ -87,62 +76,48 @@ jobs:
           yarn run prepare:packer
           cd packer
           zip -r cloud-build.zip build
-
       - name: Copy artifacts
         run: |
           mkdir artifacts          
           cp ../dbgate-merged/packer/cloud-build.zip artifacts/cloud-build.zip || true
-
       - name: Upload artifacts
         uses: actions/upload-artifact@v4
         with:
-          name: ${{ matrix.os }}
+          name: '${{ matrix.os }}'
           path: artifacts
-
       - name: Release
         uses: softprops/action-gh-release@v1
-        if: startsWith(github.ref, 'refs/tags/')
+        if: 'startsWith(github.ref, ''refs/tags/'')'
         with:
-          files: 'artifacts/**'
+          files: artifacts/**
           prerelease: true
         env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-
+          GITHUB_TOKEN: '${{ secrets.GITHUB_TOKEN }}'
       - name: Run `packer init`
         run: |
           cd ../dbgate-merged/packer
           packer init ./aws-ubuntu.pkr.hcl
         env:
-          AWS_ACCESS_KEY_ID: ${{secrets.AWS_ACCESS_KEY_ID}}
-          AWS_SECRET_ACCESS_KEY: ${{secrets.AWS_SECRET_ACCESS_KEY}}
-          AWS_DEFAULT_REGION: ${{secrets.AWS_DEFAULT_REGION}}
-
+          AWS_ACCESS_KEY_ID: '${{secrets.AWS_ACCESS_KEY_ID}}'
+          AWS_SECRET_ACCESS_KEY: '${{secrets.AWS_SECRET_ACCESS_KEY}}'
+          AWS_DEFAULT_REGION: '${{secrets.AWS_DEFAULT_REGION}}'
       - name: Run `packer build`
         run: |
           cd ../dbgate-merged/packer
           packer build ./aws-ubuntu.pkr.hcl
         env:
-          AWS_ACCESS_KEY_ID: ${{secrets.AWS_ACCESS_KEY_ID}}
-          AWS_SECRET_ACCESS_KEY: ${{secrets.AWS_SECRET_ACCESS_KEY}}
-          AWS_DEFAULT_REGION: ${{secrets.AWS_DEFAULT_REGION}}
-
-      # - name: Install AWS CLI
-      #   run: |
-      #     curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
-      #     unzip awscliv2.zip
-      #     sudo ./aws/install
-      #     sudo apt-get install jq -y
-
+          AWS_ACCESS_KEY_ID: '${{secrets.AWS_ACCESS_KEY_ID}}'
+          AWS_SECRET_ACCESS_KEY: '${{secrets.AWS_SECRET_ACCESS_KEY}}'
+          AWS_DEFAULT_REGION: '${{secrets.AWS_DEFAULT_REGION}}'
       - name: Install jq
         run: |
           sudo apt-get install jq -y
-
       - name: Delete old AMIs
         run: |
           cd ../dbgate-merged/packer
           chmod +x delete-old-amis.sh
           ./delete-old-amis.sh
         env:
-          AWS_ACCESS_KEY_ID: ${{secrets.AWS_ACCESS_KEY_ID}}
-          AWS_SECRET_ACCESS_KEY: ${{secrets.AWS_SECRET_ACCESS_KEY}}
-          AWS_DEFAULT_REGION: ${{secrets.AWS_DEFAULT_REGION}}
+          AWS_ACCESS_KEY_ID: '${{secrets.AWS_ACCESS_KEY_ID}}'
+          AWS_SECRET_ACCESS_KEY: '${{secrets.AWS_SECRET_ACCESS_KEY}}'
+          AWS_DEFAULT_REGION: '${{secrets.AWS_DEFAULT_REGION}}'

.github/workflows/build-docker-pro.yaml

@@ -1,35 +1,32 @@
+# --------------------------------------------------------------------------------------------
+# This file is generated. Do not edit manually
+# --------------------------------------------------------------------------------------------
 name: Docker image PREMIUM
-
-on:
+'on':
   push:
     tags:
       - 'v[0-9]+.[0-9]+.[0-9]+'
       - 'v[0-9]+.[0-9]+.[0-9]+-premium-beta.[0-9]+'
-
 jobs:
   build:
-
-    runs-on: ${{ matrix.os }}
-
+    runs-on: '${{ matrix.os }}'
     strategy:
       matrix:
-        os: [ubuntu-22.04]
-
+        os:
+          - ubuntu-22.04
     steps:
       - name: Context
         env:
-          GITHUB_CONTEXT: ${{ toJson(github) }}
+          GITHUB_CONTEXT: '${{ toJson(github) }}'
         run: echo "$GITHUB_CONTEXT"
       - uses: actions/checkout@v2
         with:
           fetch-depth: 1
-
       - name: Docker meta
         id: meta
         uses: docker/metadata-action@v4
         with:
-          images: |
-            dbgate/dbgate-premium
+          images: dbgate/dbgate-premium
           flavor: |
             latest=false
           tags: |
@@ -37,19 +34,16 @@ jobs:
 
             type=match,pattern=\d+.\d+.\d+,enable=${{ !contains(github.ref_name, '-docker.') && !contains(github.ref_name, '-beta.') }}
             type=raw,value=latest,enable=${{ !contains(github.ref_name, '-docker.') && !contains(github.ref_name, '-beta.') }}
-
       - name: Use Node.js 18.x
         uses: actions/setup-node@v1
         with:
           node-version: 18.x
-
       - name: Checkout dbgate/dbgate-pro
         uses: actions/checkout@v2
         with:
           repository: dbgate/dbgate-pro
-          token: ${{ secrets.GH_TOKEN }}
+          token: '${{ secrets.GH_TOKEN }}'
           path: dbgate-pro
-
       - name: Merge dbgate/dbgate-pro
         run: |
           mkdir ../dbgate-pro
@@ -61,50 +55,51 @@ jobs:
           yarn
           node sync.js --nowatch
           cd ..
-
-      - name: yarn adjustPackageJson
+      - name: adjustPackageJson
         run: |
           cd ..
           cd dbgate-merged
-          yarn adjustPackageJson
 
+          node adjustPackageJson --premium
       - name: yarn install
         run: |
           cd ..
           cd dbgate-merged
-          yarn install
 
+          # yarn --version
+          # yarn config set network-timeout 300000
+          yarn install
       - name: setCurrentVersion
         run: |
           cd ..
           cd dbgate-merged
+
           yarn setCurrentVersion
-          
       - name: printSecrets
         run: |
           cd ..
           cd dbgate-merged
-            yarn printSecrets
+
+          yarn printSecrets
         env:
-          GIST_UPLOAD_SECRET : ${{secrets.GIST_UPLOAD_SECRET}}
+          GIST_UPLOAD_SECRET: '${{secrets.GIST_UPLOAD_SECRET}}'
       - name: Prepare docker image
         run: |
           cd ..
           cd dbgate-merged
-          yarn run prepare:docker
 
+          yarn run prepare:docker
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v2
       - name: Login to DockerHub
         uses: docker/login-action@v2
         with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_PASSWORD }}
-
+          username: '${{ secrets.DOCKER_USERNAME }}'
+          password: '${{ secrets.DOCKER_PASSWORD }}'
       - name: Build and push
         uses: docker/build-push-action@v3
         with:
           push: true
           context: ../dbgate-merged/docker
-          tags: ${{ steps.meta.outputs.tags }}
-          platforms: linux/amd64,linux/arm64
+          tags: '${{ steps.meta.outputs.tags }}'
+          platforms: 'linux/amd64,linux/arm64'

.github/workflows/build-docker.yaml

@@ -1,35 +1,32 @@
-name: Docker image
-
-on:
+# --------------------------------------------------------------------------------------------
+# This file is generated. Do not edit manually
+# --------------------------------------------------------------------------------------------
+name: Docker image Community
+'on':
   push:
     tags:
       - 'v[0-9]+.[0-9]+.[0-9]+'
       - 'v[0-9]+.[0-9]+.[0-9]+-beta.[0-9]+'
-
 jobs:
   build:
-
-    runs-on: ${{ matrix.os }}
-
+    runs-on: '${{ matrix.os }}'
     strategy:
       matrix:
-        os: [ubuntu-22.04]
-
+        os:
+          - ubuntu-22.04
     steps:
       - name: Context
         env:
-          GITHUB_CONTEXT: ${{ toJson(github) }}
+          GITHUB_CONTEXT: '${{ toJson(github) }}'
         run: echo "$GITHUB_CONTEXT"
       - uses: actions/checkout@v2
         with:
           fetch-depth: 1
-
       - name: Docker meta
         id: meta
         uses: docker/metadata-action@v4
         with:
-          images: |
-            dbgate/dbgate
+          images: dbgate/dbgate
           flavor: |
             latest=false
           tags: |
@@ -37,7 +34,6 @@ jobs:
 
             type=match,pattern=\d+.\d+.\d+,enable=${{ !contains(github.ref_name, '-docker.') && !contains(github.ref_name, '-beta.') }}
             type=raw,value=latest,enable=${{ !contains(github.ref_name, '-docker.') && !contains(github.ref_name, '-beta.') }}
-
       - name: Docker alpine meta
         id: alpmeta
         uses: docker/metadata-action@v4
@@ -51,56 +47,53 @@ jobs:
 
             type=match,pattern=\d+.\d+.\d+,suffix=-alpine,enable=${{ !contains(github.ref_name, '-docker.') && !contains(github.ref_name, '-beta.') }}
             type=raw,value=alpine,enable=${{ !contains(github.ref_name, '-docker.') && !contains(github.ref_name, '-beta.') }}
-
       - name: Use Node.js 18.x
         uses: actions/setup-node@v1
         with:
           node-version: 18.x
-
-      - name: yarn adjustPackageJson
+      - name: adjustPackageJson
         run: |
-          yarn adjustPackageJson
 
+          node adjustPackageJson --community
       - name: yarn install
         run: |
+
           # yarn --version
           # yarn config set network-timeout 300000
           yarn install
-
       - name: setCurrentVersion
         run: |
+
           yarn setCurrentVersion
-          
       - name: printSecrets
         run: |
-            yarn printSecrets
+
+          yarn printSecrets
         env:
-          GIST_UPLOAD_SECRET : ${{secrets.GIST_UPLOAD_SECRET}}
+          GIST_UPLOAD_SECRET: '${{secrets.GIST_UPLOAD_SECRET}}'
       - name: Prepare docker image
         run: |
-          yarn run prepare:docker
 
+          yarn run prepare:docker
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v2
       - name: Login to DockerHub
         uses: docker/login-action@v2
         with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_PASSWORD }}
-
+          username: '${{ secrets.DOCKER_USERNAME }}'
+          password: '${{ secrets.DOCKER_PASSWORD }}'
       - name: Build and push
         uses: docker/build-push-action@v3
         with:
           push: true
           context: ./docker
-          tags: ${{ steps.meta.outputs.tags }}
-          platforms: linux/amd64,linux/arm64,linux/arm/v7
-
+          tags: '${{ steps.meta.outputs.tags }}'
+          platforms: 'linux/amd64,linux/arm64,linux/arm/v7'
       - name: Build and push alpine
         uses: docker/build-push-action@v3
         with:
           push: true
           context: ./docker
           file: ./docker/Dockerfile-alpine
-          tags: ${{ steps.alpmeta.outputs.tags }}
-          platforms: linux/amd64,linux/arm64,linux/arm/v7
+          tags: '${{ steps.alpmeta.outputs.tags }}'
+          platforms: 'linux/amd64,linux/arm64,linux/arm/v7'

.github/workflows/build-npm-pro.yaml

@@ -1,31 +1,23 @@
+# --------------------------------------------------------------------------------------------
+# This file is generated. Do not edit manually
+# --------------------------------------------------------------------------------------------
 name: NPM packages PREMIUM
-
-# on: [push]
-
-on:
+'on':
   push:
     tags:
       - 'v[0-9]+.[0-9]+.[0-9]+'
       - 'v[0-9]+.[0-9]+.[0-9]+-alpha.[0-9]+'
-
-# on:
-#   push:
-#     branches:
-#       - production
-
 jobs:
   build:
-
-    runs-on: ${{ matrix.os }}
-
+    runs-on: '${{ matrix.os }}'
     strategy:
       matrix:
-        os: [ubuntu-22.04]
-
+        os:
+          - ubuntu-22.04
     steps:
       - name: Context
         env:
-          GITHUB_CONTEXT: ${{ toJson(github) }}
+          GITHUB_CONTEXT: '${{ toJson(github) }}'
         run: echo "$GITHUB_CONTEXT"
       - uses: actions/checkout@v2
         with:
@@ -34,14 +26,12 @@ jobs:
         uses: actions/setup-node@v1
         with:
           node-version: 18.x
-
       - name: Checkout dbgate/dbgate-pro
         uses: actions/checkout@v2
         with:
           repository: dbgate/dbgate-pro
-          token: ${{ secrets.GH_TOKEN }}
+          token: '${{ secrets.GH_TOKEN }}'
           path: dbgate-pro
-
       - name: Merge dbgate/dbgate-pro
         run: |
           mkdir ../dbgate-pro
@@ -53,59 +43,55 @@ jobs:
           yarn
           node sync.js --nowatch
           cd ..
-
       - name: adjustNpmPackageJsonPremium
         run: |
           cd ..
           cd dbgate-merged
           node adjustNpmPackageJsonPremium
-
       - name: Configure NPM token
         env:
-          NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
+          NPM_TOKEN: '${{ secrets.NPM_TOKEN }}'
         run: |
           cd ..
           cd dbgate-merged
           npm config set '//registry.npmjs.org/:_authToken' "${NPM_TOKEN}"
-
+      - name: Remove dbmodel - should be not published
+        run: |
+          cd ..
+          cd dbgate-merged
+          rm -rf packages/dbmodel
       - name: yarn install
         run: |
           cd ..
           cd dbgate-merged
           yarn install
-          
       - name: setCurrentVersion
         run: |
           cd ..
           cd dbgate-merged
           yarn setCurrentVersion
-
       - name: printSecrets
         run: |
           cd ..
           cd dbgate-merged
             yarn printSecrets
         env:
-          GIST_UPLOAD_SECRET : ${{secrets.GIST_UPLOAD_SECRET}}
-
+          GIST_UPLOAD_SECRET: '${{secrets.GIST_UPLOAD_SECRET}}'
       - name: Publish dbgate-api-premium
         run: |
           cd ..
           cd dbgate-merged/packages/api
           npm publish
-
       - name: Publish dbgate-web-premium
         run: |
           cd ..
           cd dbgate-merged/packages/web
           npm publish
-
       - name: Publish dbgate-serve-premium
         run: |
           cd ..
           cd dbgate-merged/packages/serve
           npm publish
-
       - name: Publish dbgate-plugin-cosmosdb
         run: |
           cd ..

.github/workflows/build-npm.yaml

@@ -1,31 +1,23 @@
+# --------------------------------------------------------------------------------------------
+# This file is generated. Do not edit manually
+# --------------------------------------------------------------------------------------------
 name: NPM packages
-
-# on: [push]
-
-on:
+'on':
   push:
     tags:
       - 'v[0-9]+.[0-9]+.[0-9]+'
       - 'v[0-9]+.[0-9]+.[0-9]+-alpha.[0-9]+'
-
-# on:
-#   push:
-#     branches:
-#       - production
-
 jobs:
   build:
-
-    runs-on: ${{ matrix.os }}
-
+    runs-on: '${{ matrix.os }}'
     strategy:
       matrix:
-        os: [ubuntu-22.04]
-
+        os:
+          - ubuntu-22.04
     steps:
       - name: Context
         env:
-          GITHUB_CONTEXT: ${{ toJson(github) }}
+          GITHUB_CONTEXT: '${{ toJson(github) }}'
         run: echo "$GITHUB_CONTEXT"
       - uses: actions/checkout@v2
         with:
@@ -34,37 +26,30 @@ jobs:
         uses: actions/setup-node@v1
         with:
           node-version: 18.x
-
       - name: Configure NPM token
         env:
-          NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
+          NPM_TOKEN: '${{ secrets.NPM_TOKEN }}'
         run: |
           npm config set '//registry.npmjs.org/:_authToken' "${NPM_TOKEN}"
-
       - name: yarn install
         run: |
           yarn install
-          
       - name: setCurrentVersion
         run: |
           yarn setCurrentVersion
-
       - name: printSecrets
         run: |
-            yarn printSecrets
+          yarn printSecrets
         env:
-          GIST_UPLOAD_SECRET : ${{secrets.GIST_UPLOAD_SECRET}}
-
+          GIST_UPLOAD_SECRET: '${{secrets.GIST_UPLOAD_SECRET}}'
       - name: Publish types
         working-directory: packages/types
         run: |
           npm publish
-
       - name: Publish tools
         working-directory: packages/tools
         run: |
           npm publish
-
       - name: Publish sqltree
         working-directory: packages/sqltree
         run: |
@@ -74,82 +59,66 @@ jobs:
         working-directory: packages/api
         run: |
           npm publish
-
       - name: Publish datalib
         working-directory: packages/datalib
         run: |
           npm publish
-
       - name: Publish filterparser
         working-directory: packages/filterparser
         run: |
           npm publish
-
       - name: Publish web
         working-directory: packages/web
         run: |
           npm publish
-
       - name: Publish dbgate-serve
         working-directory: packages/serve
         run: |
           npm publish
-
       - name: Publish dbmodel
         working-directory: packages/dbmodel
         run: |
           npm publish
-
       - name: Publish dbgate-plugin-csv
         working-directory: plugins/dbgate-plugin-csv
         run: |
           npm publish
-
       - name: Publish dbgate-plugin-xml
         working-directory: plugins/dbgate-plugin-xml
         run: |
           npm publish
-
       - name: Publish dbgate-plugin-excel
         working-directory: plugins/dbgate-plugin-excel
         run: |
           npm publish
-
       - name: Publish dbgate-plugin-mssql
         working-directory: plugins/dbgate-plugin-mssql
         run: |
           npm publish
-
       - name: Publish dbgate-plugin-mysql
         working-directory: plugins/dbgate-plugin-mysql
         run: |
           npm publish
-
       - name: Publish dbgate-plugin-mongo
         working-directory: plugins/dbgate-plugin-mongo
         run: |
           npm publish
-
       - name: Publish dbgate-plugin-postgres
         working-directory: plugins/dbgate-plugin-postgres
         run: |
           npm publish
-
       - name: Publish dbgate-plugin-sqlite
         working-directory: plugins/dbgate-plugin-sqlite
         run: |
           npm publish
-
       - name: Publish dbgate-plugin-redis
         working-directory: plugins/dbgate-plugin-redis
         run: |
           npm publish
-
       - name: Publish dbgate-plugin-oracle
         working-directory: plugins/dbgate-plugin-oracle
         run: |
           npm publish
-
       - name: Publish dbgate-plugin-clickhouse
         working-directory: plugins/dbgate-plugin-clickhouse
         run: |

.github/workflows/build-test-containers.yaml

@@ -0,0 +1,45 @@
+# --------------------------------------------------------------------------------------------
+# This file is generated. Do not edit manually
+# --------------------------------------------------------------------------------------------
+name: Build test containers
+'on':
+  push:
+    branches:
+      - master
+      - develop
+      - feature/**
+    paths:
+      - e2e-tests/containers/**
+jobs:
+  build-docker-ssh:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout source
+        uses: actions/checkout@v3
+      - name: Build mysql-ssh-login image
+        run: |
+          docker build -t dbgate/mysql-ssh-login:latest e2e-tests/containers/mysql-ssh-login
+      - name: Build mysql-ssh-keyfile image
+        run: |
+          docker build -t dbgate/mysql-ssh-keyfile:latest e2e-tests/containers/mysql-ssh-keyfile
+      - name: Build dex image
+        run: |
+          docker build -t dbgate/dex:latest e2e-tests/containers/dex
+      - name: Log in to GHCR
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: '${{ github.actor }}'
+          password: '${{ secrets.GITHUB_TOKEN }}'
+      - name: Push mysql-ssh-login to GHCR
+        run: |
+          docker tag dbgate/mysql-ssh-login:latest ghcr.io/dbgate/mysql-ssh-login:latest
+          docker push ghcr.io/dbgate/mysql-ssh-login:latest
+      - name: Push mysql-ssh-keyfile to GHCR
+        run: |
+          docker tag dbgate/mysql-ssh-keyfile:latest ghcr.io/dbgate/mysql-ssh-keyfile:latest
+          docker push ghcr.io/dbgate/mysql-ssh-keyfile:latest
+      - name: Push dex to GHCR
+        run: |
+          docker tag dbgate/dex:latest ghcr.io/dbgate/dex:latest
+          docker push ghcr.io/dbgate/dex:latest

.github/workflows/run-tests.yaml

@@ -1,32 +1,42 @@
+# --------------------------------------------------------------------------------------------
+# This file is generated. Do not edit manually
+# --------------------------------------------------------------------------------------------
 name: Run tests
-on:
+'on':
   push:
     branches:
       - master
       - develop
-      - 'feature/**'
-
+      - feature/**
 jobs:
-  test-runner:
+  test:
     runs-on: ubuntu-latest
-    container: node:18
-
     steps:
-      - name: Context
-        env:
-          GITHUB_CONTEXT: ${{ toJson(github) }}
-        run: echo "$GITHUB_CONTEXT"
-      - uses: actions/checkout@v2
+      - name: Use Node.js 18.x
+        uses: actions/setup-node@v1
+        with:
+          node-version: 18.x
+      - uses: actions/checkout@v3
         with:
           fetch-depth: 1
       - name: yarn install
         run: |
           yarn install
+      - name: Build packer dist for cypress
+        run: |
+          yarn prepare:packer
+      - name: yarn install cypress
+        run: |
+          cd e2e-tests
+          yarn install
+      - name: Run Cypress tests
+        run: |
+          cd e2e-tests
+          yarn test:ci
       - name: Integration tests
         run: |
           cd integration-tests
           yarn test:ci
-#          yarn wait:ci
       - name: Filter parser tests
         if: always()
         run: |
@@ -40,49 +50,77 @@ jobs:
       - uses: tanmen/jest-reporter@v1
         if: always()
         with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}
+          github-token: '${{ secrets.GITHUB_TOKEN }}'
           result-file: integration-tests/result.json
           action-name: Integration tests
       - uses: tanmen/jest-reporter@v1
         if: always()
         with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}
+          github-token: '${{ secrets.GITHUB_TOKEN }}'
           result-file: packages/filterparser/result.json
           action-name: Filter parser test results
       - uses: tanmen/jest-reporter@v1
         if: always()
         with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}
+          github-token: '${{ secrets.GITHUB_TOKEN }}'
           result-file: packages/datalib/result.json
           action-name: Datalib (perspectives) test results
-
     services:
-      postgres:
+      postgres-integr:
         image: postgres
         env:
           POSTGRES_PASSWORD: Pwd2020Db
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-
-      mysql:
-        image: mysql:8.0.18
+        options: '--health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5'
+        ports:
+          - '15000:5432'
+      mysql-integr:
+        image: 'mysql:8.0.18'
         env:
           MYSQL_ROOT_PASSWORD: Pwd2020Db
-
-      mssql:
+        ports:
+          - '15001:3306'
+      mssql-integr:
         image: mcr.microsoft.com/mssql/server
         env:
-          ACCEPT_EULA: Y
+          ACCEPT_EULA: 'Y'
           SA_PASSWORD: Pwd2020Db
           MSSQL_PID: Express
-
-      clickhouse:
-        image: bitnami/clickhouse:24.8.4
+        ports:
+          - '15002:1433'
+      clickhouse-integr:
+        image: 'bitnami/clickhouse:24.8.4'
         env:
           CLICKHOUSE_ADMIN_PASSWORD: Pwd2020Db
-
-      # cockroachdb:
-      #   image: cockroachdb/cockroach
+        ports:
+          - '15005:8123'
+      oracle-integr:
+        image: 'gvenzl/oracle-xe:21-slim'
+        env:
+          ORACLE_PASSWORD: Pwd2020Db
+        ports:
+          - '15006:1521'
+      postgres-cypress:
+        image: postgres
+        options: '--health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5'
+        env:
+          POSTGRES_PASSWORD: Pwd2020Db
+        ports:
+          - '16000:5432'
+      mysql-cypress:
+        image: 'mysql:8.0.18'
+        ports:
+          - '16004:3306'
+        env:
+          MYSQL_ROOT_PASSWORD: Pwd2020Db
+      mysql-ssh-login:
+        image: 'ghcr.io/dbgate/mysql-ssh-login:latest'
+        ports:
+          - '16006:22'
+      mysql-ssh-keyfile:
+        image: 'ghcr.io/dbgate/mysql-ssh-keyfile:latest'
+        ports:
+          - '16008:22'
+      dex:
+        image: 'ghcr.io/dbgate/dex:latest'
+        ports:
+          - '16009:5556'

.nvmrc

@@ -0,0 +1 @@
+v21.7.3

CHANGELOG.md

@@ -8,11 +8,60 @@ Builds:
  - linux - application for linux
  - win - application for Windows
 
-### Not published
+### 6.1.1
+- ADDED: Trigger support (SQL Server, PostgreSQL, MySQL, Oracle)
+- FIXED: PostgreSQL and Oracle export #970
+- FIXED: Cursor Becomes Stuck When Escaping "Case" #954
+- CHANGED: Defualt search criteria for tables are names only
+- FIXED: Search in packed list
+
+### 6.1.0
+- ADDED: Fulltext search in DB model and connections, highlight searched names
+- ADDED: Tab preview mode configuration #963
+- CHANGED: Single-click to open server connection/database + ability to configure this #959
+- ADDED: Option to align numbers to right in data grid #957
+- FIXED: Cursor Becomes Stuck When Escaping "Case" #954
+- ADDED: Postgres GEOGRAPHY types are shown on map, event when executing query #948
+- FIXED: Error displaying CLOB and NCLOB in Oracle
+- FIXED: Analysing of foreign keys in Postgres and MS SQL, when the same FKS are used across different schemas
+- ADDED: Support of views, procedures, functions to Oracle. Added integration tests for Oracle
+- ADDED: Display "No rows" message, quick add new row
+- ADDED: Choose default database from list
+- ADDED: Default database is automatically selected on connect
+- ADDED: Apple-Silicon-only build for Mac #949
+- ADDED: Display comment into tables and column list #755
+
+### 6.0.0
 - ADDED: Order or filter the indexes for huge tables #922
 - ADDED: Empty string filters
 - CHANGED: (Premium) Workflow for new installation (used in Docker and AWS distribution)
-
+- ADDED: Show stored procedure and function parameters (MySQL, PostgreSQL, SQL Server, MariaDB) #348
+- FIXED: Selected database has changed when closing database grouped tab #983
+- ADDED: Add line break option to editor #823
+- ADDED: Order or filter the indexes for huge tables #922
+- ADDED: Preview mode for the top bar tab like vscode #767
+- ADDED: Keyboard navigatioon between connections, databases and tables
+- FIXED: Fixed some issues in connection search
+- FIXED: Schema selection in Export does not provide all schemas #924
+- CHANGED: Standardized Window menu in MacOS app
+- FIXED: Typecast ::date is treated as a parameter #925
+- FIXED: App crashes when trying to 'Open Structure' in a readonly connection #926
+- FIXED: Selected database has changed when closing database grouped tab #938
+- CHANGED: (Premium) Query designer and Query perspective designer moved to Premium editioin
+- CHANGED: (Premium) Compare database tool - many improvements, moved to Premium edition
+- ADDED: (Premium) Export DB model - exporting model to YAML folder, JSON or SQL folder
+- CHANGED: Model deployer - many improvements, support of rename missing objects
+- ADDED: (Premium) Premium NPM distribution
+- CHANGED: (Premium) Amazon Redshift driver moved to Premium edition
+- ADDED: Generated API documentation https://dbgate.org/docs/apidoc.html
+- ADDED: NPM distribution now supports all dbgate database connectors, many improvements NPM packages
+- CHANGED: Optimalized size of NPM plugins (eg. dbgate-plugin-mssql from 1.34 MB to 71 kB)
+- CHANGED: Unsaved connections are now shown in "Recent and unsaved" folder after disconnect
+- FIXED: Correctly show focused control, as defined by UX standards
+- ADDED: Data duplicator - weak references
+- ADDED: View JSON detail of log messages from export/import jobs and query executions
+- ADDED: Rename procedure/function context menu
+- ADDED: Show SQL quick view
 
 ### 5.5.6
 - FIXED: DbGate process consumes 100% after UI closed - Mac, Linux (#917, #915)

README.md

@@ -17,7 +17,7 @@ DbGate is licensed under GPL-3.0 license and is free to use for any purpose.
 * Try it online - [demo.dbgate.org](https://demo.dbgate.org) - online demo application
 * **Download** application for Windows, Linux or Mac from [dbgate.org](https://dbgate.org/download/)
 * Run web version as [NPM package](https://www.npmjs.com/package/dbgate-serve) or as [docker image](https://hub.docker.com/r/dbgate/dbgate)
-* Use nodeJs [scripting interface](https://dbgate.org/docs/scripting.html) ([API documentation](https://dbgate.org/docs/apidoc.html))
+* Use nodeJs [scripting interface](https://dbgate.org/docs/scripting) ([API documentation](https://dbgate.org/docs/apidoc))
 * [Recommend DbGate](https://testimonial.to/dbgate) | [Rate on G2](https://www.g2.com/products/dbgate/reviews)
 
 ## Supported databases
@@ -87,7 +87,7 @@ Any contributions are welcome. If you want to contribute without coding, conside
 * Create issue, if you find problem in app, or you have idea to new feature. If issue already exists, you could leave comment on it, to prioritise most wanted issues
 * Create some tutorial video on [youtube](https://www.youtube.com/playlist?list=PLCo7KjCVXhr0RfUSjM9wJMsp_ShL1q61A)
 * Become a backer on [GitHub sponsors](https://github.com/sponsors/dbgate) or [Open collective](https://opencollective.com/dbgate)
-* Where a small coding is acceptable for you, you could [create plugin](https://dbgate.org/docs/plugin-development.html). Plugins for new themes can be created actually without JS coding
+* Where a small coding is acceptable for you, you could [create plugin](https://dbgate.org/docs/plugin-development). Plugins for new themes can be created actually without JS coding
 
 Thank you!
 

adjustPackageJson.js

@@ -41,6 +41,29 @@ function adjustFile(file, isApp = false) {
     delete json.optionalDependencies.msnodesqlv8;
   }
 
+  if (process.argv.includes('--community')) {
+    delete json.optionalDependencies['mongodb-client-encryption'];
+  }
+
+  if (isApp && process.argv.includes('--premium')) {
+    json.build.win.target = [
+      {
+        target: 'nsis',
+        arch: ['x64'],
+      },
+    ];
+    json.build.linux.target = [
+      {
+        target: 'AppImage',
+        arch: ['x64'],
+      },
+    ];
+    json.name = 'dbgate-premium';
+    json.build.artifactName = 'dbgate-premium-${version}-${os}_${arch}.${ext}';
+    json.build.appId = 'org.dbgate.premium';
+    json.build.productName = 'DbGate Premium';
+  }
+
   fs.writeFileSync(file, JSON.stringify(json, null, 2), 'utf-8');
 }
 

app/package.json

@@ -1,6 +1,6 @@
 {
   "name": "dbgate",
-  "version": "5.0.0-alpha.1",
+  "version": "6.0.0-alpha.1",
   "private": true,
   "author": "Jan Prochazka <jenasoft.database@gmail.com>",
   "description": "Opensource database administration tool",
@@ -19,7 +19,6 @@
     "artifactName": "dbgate-${version}-${os}_${arch}.${ext}",
     "appId": "org.dbgate",
     "productName": "DbGate",
-    "afterSign": "electron-builder-notarize",
     "asarUnpack": "**/*.node",
     "mac": {
       "category": "database",
@@ -38,9 +37,11 @@
         "target": "default",
         "arch": [
           "universal",
-          "x64"
+          "x64",
+          "arm64"
         ]
-      }
+      },
+      "notarize": true
     },
     "linux": {
       "target": [
@@ -129,7 +130,6 @@
     "copyfiles": "^2.2.0",
     "cross-env": "^6.0.3",
     "electron": "30.0.2",
-    "electron-builder": "23.1.0",
-    "electron-builder-notarize": "^1.5.2"
+    "electron-builder": "25.1.8"
   }
 }

app/src/electron.js

@@ -108,7 +108,7 @@ function commandItem(item) {
 }
 
 function buildMenu() {
-  let template = _cloneDeepWith(mainMenuDefinition({ editMenu: true }), item => {
+  let template = _cloneDeepWith(mainMenuDefinition({ editMenu: true, isMac: isMac() }), item => {
     if (item.divider) {
       return { type: 'separator' };
     }

app/src/mainMenuDefinition.js

@@ -1,4 +1,4 @@
-module.exports = ({ editMenu }) => [
+module.exports = ({ editMenu, isMac }) => [
   {
     label: 'File',
     submenu: [
@@ -24,20 +24,6 @@ module.exports = ({ editMenu }) => [
       { command: 'app.disconnect', hideDisabled: true, skipInApp: true },
     ],
   },
-  {
-    label: 'Window',
-    submenu: [
-      { command: 'tabs.closeTab', hideDisabled: false },
-      { command: 'tabs.closeAll', hideDisabled: false },
-      { command: 'tabs.closeTabsWithCurrentDb', hideDisabled: false },
-      { command: 'tabs.closeTabsButCurrentDb', hideDisabled: false },
-      { divider: true },
-      { command: 'app.zoomIn', hideDisabled: true },
-      { command: 'app.zoomOut', hideDisabled: true },
-      { command: 'app.zoomReset', hideDisabled: true },
-    ],
-  },
-
   editMenu
     ? {
         label: 'Edit',
@@ -75,6 +61,15 @@ module.exports = ({ editMenu }) => [
       { divider: true },
       { command: 'theme.changeTheme', hideDisabled: true },
       { command: 'settings.show' },
+      { divider: true },
+      { command: 'tabs.closeTab', hideDisabled: false },
+      { command: 'tabs.closeAll', hideDisabled: false },
+      { command: 'tabs.closeTabsWithCurrentDb', hideDisabled: false },
+      { command: 'tabs.closeTabsButCurrentDb', hideDisabled: false },
+      { divider: true },
+      { command: 'app.zoomIn', hideDisabled: true },
+      { command: 'app.zoomOut', hideDisabled: true },
+      { command: 'app.zoomReset', hideDisabled: true },
     ],
   },
   {
@@ -94,6 +89,14 @@ module.exports = ({ editMenu }) => [
       { command: 'app.resetSettings', hideDisabled: true },
     ],
   },
+  ...(isMac
+    ? [
+        {
+          role: 'window',
+          submenu: [{ role: 'minimize' }, { role: 'zoom' }, { type: 'separator' }, { role: 'front' }],
+        },
+      ]
+    : []),
   {
     label: 'Help',
     submenu: [

common/defineVolatileDependencies.js

@@ -14,7 +14,7 @@ for (const pkg of volatilePackages) {
   if (apiPackageJson.dependencies[pkg]) {
     dependencies[pkg] = apiPackageJson.dependencies[pkg];
   }
-  if (apiPackageJson.optionalDependencies[pkg]) {
+  if (apiPackageJson.optionalDependencies?.[pkg]) {
     optionalDependencies[pkg] = apiPackageJson.optionalDependencies[pkg];
   }
 }

common/processWorkflows.js

@@ -0,0 +1,174 @@
+const fs = require('fs');
+const path = require('path');
+const yaml = require('js-yaml');
+const _ = require('lodash');
+
+const indir = path.resolve(path.join(__dirname, '..', 'workflow-templates'));
+const outdir = path.resolve(path.join(__dirname, '..', '.github', 'workflows'));
+
+const includes = {};
+
+const HEADER = `# --------------------------------------------------------------------------------------------
+# This file is generated. Do not edit manually
+# --------------------------------------------------------------------------------------------
+`;
+
+function readIncludes() {
+  for (const file of fs.readdirSync(indir)) {
+    const text = fs.readFileSync(path.join(indir, file), { encoding: 'utf-8' });
+    const json = yaml.load(text);
+    if (json._module) {
+      for (const key in json) {
+        if (key === '_module') {
+          continue;
+        }
+        includes[key] = json[key];
+      }
+    }
+  }
+}
+
+let modified = false;
+
+function conditionMatch(condition, args) {
+  if (_.isString(condition)) {
+    return args.defs.includes(condition);
+  }
+  return false;
+}
+
+function processJsonStep(json, args) {
+  return _.cloneDeepWith(json, value => {
+    if (_.isArray(value)) {
+      const res = [];
+      let arrayModified = false;
+      for (const item of value) {
+        if (item._if) {
+          modified = true;
+          arrayModified = true;
+          if (conditionMatch(item._if, args)) {
+            res.push(_.omit(item, ['_if']));
+          }
+        } else if (item._replace || item._include) {
+          const replaceWith = item._replace ? args.replace?.[item._replace] : includes[item._include];
+          if (replaceWith) {
+            modified = true;
+            arrayModified = true;
+            if (_.isArray(replaceWith)) {
+              res.push(...replaceWith);
+            } else {
+              res.push(replaceWith);
+            }
+          } else {
+            res.push(item);
+          }
+        } else {
+          res.push(item);
+        }
+      }
+      if (arrayModified) {
+        return res;
+      }
+      return undefined;
+    }
+
+    if (_.isPlainObject(value)) {
+      if (_.intersection(args.allDefs ?? [], Object.keys(value))?.length > 0) {
+        modified = true;
+        for (const key in value) {
+          if (args.defs.includes(key)) {
+            return value[key];
+          }
+        }
+        return undefined;
+      }
+    }
+
+    if (_.isString(value)) {
+      let stringModified = false;
+      for (const key of Object.keys(args.stringReplace ?? {})) {
+        if (value.includes(key)) {
+          modified = true;
+          stringModified = true;
+          value = value.replaceAll(key, args.stringReplace[key]);
+        }
+      }
+      if (stringModified) {
+        return value;
+      }
+      return undefined;
+    }
+
+    if (value?._include) {
+      modified = true;
+      return includes[value?._include];
+    }
+
+    if (value?._replace) {
+      modified = true;
+      return args?.replace[value?._replace];
+    }
+  });
+}
+
+function processJson(json, args = {}) {
+  const MAX_STEPS = 64;
+  for (let i = 0; i < MAX_STEPS; i++) {
+    modified = false;
+    json = processJsonStep(json, args);
+    if (!modified) {
+      break;
+    }
+  }
+  return json;
+}
+
+function processFiles() {
+  const dumpOptions = {
+    lineWidth: -1,
+  };
+  for (const file of fs.readdirSync(indir)) {
+    const text = fs.readFileSync(path.join(indir, file), { encoding: 'utf-8' });
+    const json = yaml.load(text);
+
+    if (json._module) {
+      continue;
+    }
+
+    if (json._templates) {
+      const allDefs = Object.keys(json._templates);
+      for (const key in json._templates) {
+        allDefs.push(...(json._templates[key].defs ?? []));
+      }
+
+      for (const key in json._templates) {
+        const args = {
+          defs: [key, ...(json._templates[key]?.defs ?? [])],
+          replace: json._templates[key]?.replace,
+          stringReplace: json._templates[key]?.['string-replace'],
+          allDefs,
+        };
+        const converted = processJson(_.omit(json, ['_templates']), args);
+        const out = path.join(outdir, json._templates[key].file);
+        fs.writeFileSync(out, HEADER + yaml.dump(converted, dumpOptions));
+      }
+    } else {
+      fs.writeFileSync(path.join(outdir, file), HEADER + yaml.dump(processJson(json), dumpOptions));
+    }
+  }
+}
+
+function deleteOldFiles() {
+  const files = fs.readdirSync(outdir);
+  for (const file of files) {
+    fs.unlinkSync(path.join(outdir, file));
+  }
+}
+
+function run() {
+  deleteOldFiles();
+  readIncludes();
+  processFiles();
+}
+
+run();

common/volatilePackages.js

@@ -19,6 +19,7 @@ const volatilePackages = [
   'activedirectory2',
   'axios',
   'ssh2',
+  'wkx',
 ];
 
 module.exports = volatilePackages;

e2e-tests/containers/dex/Dockerfile

@@ -0,0 +1,7 @@
+FROM dexidp/dex:v2.35.3
+
+COPY dex-config.yaml /etc/dex/cfg/config.yaml
+
+EXPOSE 5556
+
+CMD ["dex", "serve", "/etc/dex/cfg/config.yaml"]

e2e-tests/containers/dex/dex-config.yaml

@@ -0,0 +1,22 @@
+issuer: http://localhost:16009/dex
+storage:
+  type: memory
+web:
+  http: 0.0.0.0:5556
+
+oauth2:
+  skipApprovalScreen: true
+
+staticClients:
+  - id: my-app
+    redirectURIs:
+      - 'http://localhost:3000/'
+    name: 'My Test App'
+    secret: my-secret
+
+enablePasswordDB: true
+staticPasswords:
+  - email: "test@example.com"
+    hash: "$2y$10$JcmlXnV1y7.egUdKwYNbseOnqYVIGc323gtvvHh4ZuSPZB30veYZy"
+    username: "test"
+    userID: "1234"

e2e-tests/containers/mysql-ssh-keyfile/Dockerfile

@@ -0,0 +1,27 @@
+FROM ubuntu:22.04
+
+RUN apt-get update && \
+    apt-get install -y openssh-server mysql-server && \
+    apt-get clean && rm -rf /var/lib/apt/lists/*
+
+RUN mkdir /var/run/sshd
+
+RUN sed -i 's/#PasswordAuthentication yes/PasswordAuthentication no/' /etc/ssh/sshd_config && \
+    sed -i 's/#PermitRootLogin prohibit-password/PermitRootLogin without-password/' /etc/ssh/sshd_config
+
+RUN mkdir -p /root/.ssh && chmod 700 /root/.ssh
+
+COPY mykey.pub /root/.ssh/authorized_keys
+
+RUN chmod 600 /root/.ssh/authorized_keys
+
+RUN service mysql start && \
+    mysql -uroot -e "ALTER USER 'root'@'localhost' IDENTIFIED WITH mysql_native_password BY 'root'; FLUSH PRIVILEGES;" && \
+    service mysql stop
+
+EXPOSE 22
+
+COPY start.sh /start.sh
+RUN chmod +x /start.sh
+
+CMD ["/start.sh"]

e2e-tests/containers/mysql-ssh-keyfile/mykey.pub

@@ -0,0 +1 @@
+ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQClN/tu+X7PVYpMBEQDwxtyG0oNuKqcJdBud/vBUalyYSpliXO30BsBusjCyAh8ENVllmGTRaTcqAUPg/vwekaAMFZu9EXmQsdhPEWSxrkPkXXKsWrZVPtRV8l3wuSy/VLxbXRLujTpIEE5Au88e2CFN3zzuhg9rvsXE0cGjyYe9Cw3Ub+Yo6rbnESI7F+Jun29/wD8RF+kRL5cr7BrDoOBQf4yGexrTbUPMZ4b8QlGfXUwZhn6LlWVoWq7BMRelnfIAkuC+LwavvnkMqXMITmGa7sG0zOXByW7iGdP6TWVq9Ks+yCZoAA9ncAkgkpwoNrOSXR6dM5UjrsfJ2Na332QfxK8MmC+nBiyXIKUaQY2G8Wfm93GQqDc+349JFwH4ZjgYWJlTd/UfRCn4j30jxe1RKnjt5xGqb3L+j8JfuSPcojWRekg/K8GxBmaMCcEdqjIFD4MxeIUW3bGHd30wnuZkTPWlSmAGrO0fZXj0s1fmOBME05TjE8jsyKPmb2Kd3JdtCuTrjOgrYiHaMfcv/VqVhIv2kCKjnvhAxCqfGfKxxgcerItbrsfFAvYak1kwZelBquHOK6SkMH5WhDyhvGY+ZF0Wbg3CpynlpJVu4E3LSglYNbkIWqS3pAf+hqJtXl9SHCLgKFUcCIz0ZNQkT+PtUlR3klp6qoPXUWrcL3pdQ== test

e2e-tests/containers/mysql-ssh-keyfile/start.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+service ssh start
+
+service mysql start
+
+tail -f /dev/null

e2e-tests/containers/mysql-ssh-login/Dockerfile

@@ -0,0 +1,23 @@
+FROM ubuntu:22.04
+
+RUN apt-get update && \
+    apt-get install -y openssh-server mysql-server && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+RUN mkdir /var/run/sshd
+
+RUN sed -i 's/#PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config
+RUN echo 'root:root' | chpasswd
+
+RUN service mysql start && \
+    mysql -uroot -e "ALTER USER 'root'@'localhost' IDENTIFIED WITH mysql_native_password BY 'root'; FLUSH PRIVILEGES;" && \
+    service mysql stop
+
+EXPOSE 22
+
+COPY start.sh /start.sh
+
+RUN chmod +x /start.sh
+
+CMD ["/start.sh"]

e2e-tests/containers/mysql-ssh-login/start.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+service ssh start
+
+service mysql start
+
+tail -f /dev/null

e2e-tests/cypress.config.js

@@ -0,0 +1,44 @@
+const { defineConfig } = require('cypress');
+const killPort = require('kill-port');
+const { clearTestingData } = require('./e2eTestTools');
+const waitOn = require('wait-on');
+const { exec } = require('child_process');
+
+module.exports = defineConfig({
+  e2e: {
+    setupNodeEvents(on, config) {
+      // implement node event listeners here
+
+      on('before:spec', async details => {
+        await clearTestingData();
+        // console.log('********************* DETAILS *********************', JSON.stringify(details));
+
+        if (config.isInteractive) {
+          await killPort(3000);
+          switch (details.fileName) {
+            case 'add-connection':
+              serverProcess = exec('yarn start:add-connection');
+              break;
+            case 'portal':
+              serverProcess = exec('yarn start:portal');
+              break;
+            case 'oauth':
+              serverProcess = exec('yarn start:oauth');
+              break;
+            case 'browse-data':
+              serverProcess = exec('yarn start:browse-data');
+              break;
+          }
+
+          await waitOn({ resources: ['http://localhost:3000'] });
+          serverProcess.stdout.on('data', data => {
+            console.log(data.toString());
+          });
+          serverProcess.stderr.on('data', data => {
+            console.error(data.toString());
+          });
+        }
+      });
+    },
+  },
+});

e2e-tests/cypress/e2e/add-connection.cy.js

@@ -0,0 +1,109 @@
+const path = require('path');
+
+describe('Add connection', () => {
+  it('successfully loads', () => {
+    cy.visit('http://localhost:3000');
+    cy.contains('Database not selected');
+  });
+
+  it('adds connection', () => {
+    cy.visit('http://localhost:3000');
+    // cy.get('[data-testid=ConnectionList_buttonNewConnection]').click();
+    cy.get('[data-testid=ConnectionDriverFields_connectionType]').select('MySQL');
+    cy.get('[data-testid=ConnectionDriverFields_user]').clear().type('root');
+    cy.get('[data-testid=ConnectionDriverFields_password]').clear().type('Pwd2020Db');
+    cy.get('[data-testid=ConnectionDriverFields_port]').clear().type('16004');
+    cy.get('[data-testid=ConnectionDriverFields_displayName]').clear().type('test-mysql-1');
+
+    // test connection
+    cy.get('[data-testid=ConnectionTab_buttonTest]').click();
+    cy.contains('Connected:');
+
+    // save and connect
+    cy.get('[data-testid=ConnectionTab_buttonSave]').click();
+    cy.get('[data-testid=ConnectionTab_buttonConnect]').click();
+    cy.contains('performance_schema');
+  });
+
+  it('SSH connection - user + password', () => {
+    cy.visit('http://localhost:3000');
+    cy.contains('Connections');
+
+    // cy.realPress('F1');
+    // cy.realType('Close all');
+    // cy.realPress('Enter');
+
+    cy.get('[data-testid=ConnectionList_buttonNewConnection]').click();
+    cy.get('[data-testid=ConnectionDriverFields_connectionType]').select('MySQL');
+    cy.get('[data-testid=ConnectionDriverFields_user]').clear().type('root');
+    cy.get('[data-testid=ConnectionDriverFields_password]').clear().type('root');
+
+    cy.get('[data-testid=ConnectionDriverFields_displayName]').clear().type('test-mysql-ssh-1');
+
+    cy.get('[data-testid=ConnectionTab_tabSshTunnel]').click();
+    cy.get('[data-testid=ConnectionSshTunnelFields_useSshTunnel]').check();
+    cy.get('[data-testid=ConnectionSshTunnelFields_sshLogin]').clear().type('root');
+    cy.get('[data-testid=ConnectionSshTunnelFields_sshPassword]').clear().type('root');
+    cy.get('[data-testid=ConnectionSshTunnelFields_sshPort]').clear().type('16006');
+    cy.get('[data-testid=ConnectionTab_buttonSave]').click();
+    cy.get('[data-testid=ConnectionTab_buttonConnect]').click();
+    cy.contains('performance_schema');
+  });
+
+  it('SSH connection - keyfile', () => {
+    cy.visit('http://localhost:3000');
+    cy.contains('Connections');
+
+    // cy.realPress('F1');
+    // cy.realType('Close all');
+    // cy.realPress('Enter');
+
+    cy.get('[data-testid=ConnectionList_buttonNewConnection]').click();
+    cy.get('[data-testid=ConnectionDriverFields_connectionType]').select('MySQL');
+    cy.get('[data-testid=ConnectionDriverFields_user]').clear().type('root');
+    cy.get('[data-testid=ConnectionDriverFields_password]').clear().type('root');
+
+    cy.get('[data-testid=ConnectionDriverFields_displayName]').clear().type('test-mysql-ssh-2');
+
+    cy.get('[data-testid=ConnectionTab_tabSshTunnel]').click();
+    cy.get('[data-testid=ConnectionSshTunnelFields_useSshTunnel]').check();
+    cy.get('[data-testid=ConnectionSshTunnelFields_sshMode]').select('Key file');
+    cy.get('[data-testid=ConnectionSshTunnelFields_sshLogin]').clear();
+    cy.get('[data-testid=ConnectionSshTunnelFields_sshLogin]').type('root');
+    cy.get('[data-testid=ConnectionSshTunnelFields_sshKeyfile]')
+      .clear()
+      .type(path.join(Cypress.config('fileServerFolder'), 'cypress', 'e2e', 'mykey'));
+    cy.get('[data-testid=ConnectionSshTunnelFields_sshPort]').clear().type('16008');
+    cy.get('[data-testid=ConnectionTab_buttonSave]').click();
+    cy.get('[data-testid=ConnectionTab_buttonConnect]').click();
+    cy.contains('performance_schema');
+  });
+
+  it('ask password - mysql', () => {
+    cy.visit('http://localhost:3000');
+
+    cy.get('[data-testid=ConnectionList_buttonNewConnection]').click();
+    cy.get('[data-testid=ConnectionDriverFields_connectionType]').select('MySQL');
+    cy.get('[data-testid=ConnectionDriverFields_user]').clear().type('root');
+    cy.get('[data-testid=ConnectionDriverFields_password]').clear().type('Pwd2020Db');
+    cy.get('[data-testid=ConnectionDriverFields_port]').clear().type('16004');
+    cy.get('[data-testid=ConnectionDriverFields_displayName]').clear().type('test-mysql-2');
+    cy.testid('ConnectionDriverFields_passwordMode').select('askPassword');
+
+    // test connection
+    cy.get('[data-testid=ConnectionTab_buttonTest]').click();
+    cy.testid('DatabaseLoginModal_password').clear().type('Pwd2020Db');
+    cy.testid('DatabaseLoginModal_connect').click();
+
+    cy.contains('Connected:');
+
+    cy.get('[data-testid=ConnectionTab_buttonSave]').click();
+    cy.get('[data-testid=ConnectionTab_buttonConnect]').click();
+
+    // again type DB password - not saved
+    cy.testid('DatabaseLoginModal_password').clear().type('Pwd2020Db');
+    cy.testid('DatabaseLoginModal_connect').click();
+
+    cy.contains('performance_schema');
+  });
+});

e2e-tests/cypress/e2e/browse-data.cy.js

@@ -0,0 +1,43 @@
+describe('Data browser data', () => {
+  it('Load table data', () => {
+    cy.visit('http://localhost:3000');
+
+    cy.contains('MySql-connection').click();
+    cy.contains('Chinook').click();
+    cy.contains('Album').click();
+    cy.contains('Let There Be Rock').click();
+    cy.contains('Rows: 347');
+    cy.realPress(['Control', 'ArrowRight']);
+    cy.contains('Aerosmith');
+  });
+
+  it.only('Filter model', () => {
+    cy.visit('http://localhost:3000');
+
+    cy.contains('MySql-connection').click();
+    cy.contains('Chinook').click();
+    cy.testid('SqlObjectList_search').clear().type('album');
+    cy.contains('Tables (1/11)');
+    cy.contains('347 rows, InnoDB');
+    cy.testid('SqlObjectList_searchMenuDropDown').click();
+    cy.contains('Column name').click();
+    cy.contains('Tables (2/11)');
+    cy.contains('AlbumId');
+    cy.contains('Column name').click();
+    cy.contains('AlbumId').should('not.exist');
+    cy.contains('Tables (1/11)');
+  });
+
+  it('Show raw data', () => {
+    cy.visit('http://localhost:3000');
+
+    cy.contains('MySql-connection').click();
+    cy.contains('Chinook').click();
+    cy.contains('Album').rightclick();
+    cy.contains('Open raw data').click();
+    cy.contains('Let There Be Rock').click();
+    cy.contains('Rows: 347').should('not.exist');
+    cy.realPress(['Control', 'ArrowRight']);
+    cy.contains('Aerosmith').should('not.exist');
+  });
+});

e2e-tests/cypress/e2e/mykey

@@ -0,0 +1,49 @@
+-----BEGIN OPENSSH PRIVATE KEY-----
+b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAACFwAAAAdzc2gtcn
+NhAAAAAwEAAQAAAgEApTf7bvl+z1WKTAREA8MbchtKDbiqnCXQbnf7wVGpcmEqZYlzt9Ab
+AbrIwsgIfBDVZZZhk0Wk3KgFD4P78HpGgDBWbvRF5kLHYTxFksa5D5F1yrFq2VT7UVfJd8
+Lksv1S8W10S7o06SBBOQLvPHtghTd887oYPa77FxNHBo8mHvQsN1G/mKOq25xEiOxfibp9
+vf8A/ERfpES+XK+waw6DgUH+Mhnsa021DzGeG/EJRn11MGYZ+i5VlaFquwTEXpZ3yAJLgv
+i8Gr755DKlzCE5hmu7BtMzlwclu4hnT+k1lavSrPsgmaAAPZ3AJIJKcKDazkl0enTOVI67
+HydjWt99kH8SvDJgvpwYslyClGkGNhvFn5vdxkKg3Pt+PSRcB+GY4GFiZU3f1H0Qp+I99I
+8XtUSp47ecRqm9y/o/CX7kj3KI1kXpIPyvBsQZmjAnBHaoyBQ+DMXiFFt2xh3d9MJ7mZEz
+1pUpgBqztH2V49LNX5jgTBNOU4xPI7Mij5m9indyXbQrk64zoK2Ih2jH3L/1alYSL9pAio
+574QMQqnxnyscYHHqyLW67HxQL2GpNZMGXpQarhziukpDB+VoQ8obxmPmRdFm4Nwqcp5aS
+VbuBNy0oJWDW5CFqkt6QH/oaibV5fUhwi4ChVHAiM9GTUJE/j7VJUd5JaeqqD11Fq3C96X
+UAAAdAKwO+PisDvj4AAAAHc3NoLXJzYQAAAgEApTf7bvl+z1WKTAREA8MbchtKDbiqnCXQ
+bnf7wVGpcmEqZYlzt9AbAbrIwsgIfBDVZZZhk0Wk3KgFD4P78HpGgDBWbvRF5kLHYTxFks
+a5D5F1yrFq2VT7UVfJd8Lksv1S8W10S7o06SBBOQLvPHtghTd887oYPa77FxNHBo8mHvQs
+N1G/mKOq25xEiOxfibp9vf8A/ERfpES+XK+waw6DgUH+Mhnsa021DzGeG/EJRn11MGYZ+i
+5VlaFquwTEXpZ3yAJLgvi8Gr755DKlzCE5hmu7BtMzlwclu4hnT+k1lavSrPsgmaAAPZ3A
+JIJKcKDazkl0enTOVI67HydjWt99kH8SvDJgvpwYslyClGkGNhvFn5vdxkKg3Pt+PSRcB+
+GY4GFiZU3f1H0Qp+I99I8XtUSp47ecRqm9y/o/CX7kj3KI1kXpIPyvBsQZmjAnBHaoyBQ+
+DMXiFFt2xh3d9MJ7mZEz1pUpgBqztH2V49LNX5jgTBNOU4xPI7Mij5m9indyXbQrk64zoK
+2Ih2jH3L/1alYSL9pAio574QMQqnxnyscYHHqyLW67HxQL2GpNZMGXpQarhziukpDB+VoQ
+8obxmPmRdFm4Nwqcp5aSVbuBNy0oJWDW5CFqkt6QH/oaibV5fUhwi4ChVHAiM9GTUJE/j7
+VJUd5JaeqqD11Fq3C96XUAAAADAQABAAACAAOZKogKNbF8+BwbuQIpTP8Oim2uE0PUrFYb
+lYdqUznDKqc3d9ENs+jfu+DyIQKCuK2+ZAr5vf4dnVlEffIzgjbnK/ASsIsJR/UILlkPVu
+x4izY06FIqFackivtpao2zaAu2CeMJqn6r+H+uCVkp3rpIBm1QekKtqEOiZIOcuS8nZjx2
+MqliYdA9S7N9DRT0037rKjGpqOOXqQW1FdnfC7d+9nL6OY9EtqgeuTDRQ8BzRGAfeSVdE6
+QZ9eHI2CCvP5XtkYjICLYUaWDc8YYhRcggWdxrhnjIQRF1MRApJhpHubjmvJkGm7q70CQm
+FPT1q+OeRJko022SsOScI6p7qkJjEsHS4RLbSGkDAn84U9l1MnHbzOqOQbI/Mnn7vNyZt4
+e7n/ExUg9HglpdLgETl0oLIaiVgYGrJSLflfYtrgfEISGtoroQDj/X96HZsF+Aku+I3U4z
+OD7UasFDl3LsTO3bWz8/vqkCfi/ncejZ4keWUA6VahbtJUg8E/eEIWMz8gFoO/vWf1l0H0
+V51ISzInt1yVnqhdbRPH+Ce4NlNjq9JlhiZVJSsc6ke2580Okttc3JETzdHWNac7k+/0u4
+lPZK2sII1cq/d+9YAe3JVpUuiQyX6R4Crga2GIyevoqm3EyXUNN/vjDCx4qyToOtJUzHao
+tXI4tZZkBmtVrhqf1pAAABAQC3lG50iMbJXL8vVsB7P8UWLgmI13PqwHB8ldzsLrNtkU7q
+N3KYdz/ycGGPAKxfMJiGs00qT8/d107MZYmj7hHE84VuBiHVGSXbzmL5zeth0aN4oINee0
+jlvCPKjqhoImPtrU1fLglqOeTNJcZK7XzwiZry1ZHp0hXd4XPrWHZBZo37SYv9/t652zR4
+qmXlxwwwdtzuNFiAexTu0K4zY12hjmoGlvlCS2YJ+B2wJNd3E1RnL505m7dIA/dLnCLeEj
+zf8j1XiZ9i/m7mrU/g/e9SFP+HVwi5AV5eyHbNSPD6GZ7quWL4SlP+GGkNtKden+LIaB+G
+iRwCLTgCVThdIC6oAAABAQDA6Gm7luZuzOoWBQzVKc74AOf64Op+eZwQBCC3V+eHjmyIQW
+oBozvlvaNxapNxJZQjtnlckCkW59RheQxm155j/MV6aVvaD9WWNFB9k+0fujmPd4fPoZF1
+9y5J3L3l9hmv+51vZvKvky5mXxOGdONTQZkpGqGvmlZpVO9jWe4WWACokqRIrS9zwtfLgf
+hH+hUMh+9R1iIKACleNdfHbiqia+XIXxivbVdCtTgdPM6mW8i3gP1/oSTpYZI5KIRH4PTO
+6OgG8EvRCIPc/0mddXZqiabhpPM/r5qe0jKxcQHFf1siwq7EyZ/C8ctBxpEK4dVCNnCTX4
+k7+/JizOJzhzljAAABAQDbQT0BsBCM2Dl+LbwpAy6iaS0fWqhCnQ9B6ojWcfNouXuIW4tF
+6fwmoqyFU2SoORmj0G6ww1NiBr+gwSDo5wSEpCI78S1CnFcp7J1HhONGthmdYDclpDlci5
+t5AOIC7hhmdkqL46happxG9MH8oOua7/cqK7300nJo+ZS3XOo4O3siTjDVtVKXRAY5jWuG
+mLNXQ8JKCzqmFfLlAAgGWaY7rMaGu+9Rom+F4FEZ4IUitcsN77jFNBXMXqnWy/ayr8CJpe
+CxgwBm5JWQ7m+cmskMl3nmjkNr8nCiZoMcz4sIrVkZWS4PHoVFtN3jKYmo3c02kq8/378I
+jIy5WYf/zrVHAAAABHRlc3QBAgMEBQY=
+-----END OPENSSH PRIVATE KEY-----

e2e-tests/cypress/e2e/oauth.cy.js

@@ -0,0 +1,14 @@
+describe('OAuth', () => {
+  it('OAuth login', () => {
+    cy.visit('http://localhost:3000');
+
+    // login on DEX
+    cy.get('#login').clear().type('test@example.com');
+    cy.get('#password').clear().type('test');
+    cy.get('#submit-login').click();
+
+    // check DbGate connection
+    cy.contains('MySql-connection').click();
+    cy.contains('performance_schema');
+  });
+});

e2e-tests/cypress/e2e/portal.cy.js

@@ -0,0 +1,93 @@
+describe('Run as portal', () => {
+  it('successfully loads', () => {
+    cy.visit('http://localhost:3000');
+    cy.contains('MySql-connection');
+    cy.contains('Postgres-connection');
+  });
+
+  // it('Delete chinook', () => {
+  //   cy.visit('http://localhost:3000');
+
+  //   cy.contains('MySql-connection').rightclick();
+  //   cy.contains('New Query (server)').click();
+  //   cy.realType('drop database if exists Chinook');
+  //   cy.realPress('F5');
+  //   cy.contains('Query execution finished');
+
+  //   cy.contains('Postgres-connection').rightclick();
+  //   cy.contains('New Query (server)').click();
+  //   cy.realType('drop database if exists "Chinook"');
+  //   cy.realPress('F5');
+  //   cy.contains('Query execution finished');
+
+  //   // cy.realPress('F1');
+  //   // cy.realType('Close all');
+  //   // cy.realPress('Enter');
+  // });
+
+  it('Create Chinook MySQL', () => {
+    cy.visit('http://localhost:3000');
+
+    cy.contains('MySql-connection').click();
+    cy.contains('MySql-connection').rightclick();
+    cy.contains('Create database').click();
+    cy.get('[data-testid=InputTextModal_value]').clear().type('Chinook');
+    cy.get('[data-testid=InputTextModal_ok]').click();
+  });
+
+  it('Create Chinook Postgres', () => {
+    cy.visit('http://localhost:3000');
+
+    cy.contains('Postgres-connection').click();
+    cy.contains('Postgres-connection').rightclick();
+    cy.contains('Create database').click();
+    cy.get('[data-testid=InputTextModal_value]').clear().type('Chinook');
+    cy.get('[data-testid=InputTextModal_ok]').click();
+  });
+
+  it('Import Chinook MySQL', () => {
+    cy.visit('http://localhost:3000');
+    cy.contains('MySql-connection').click();
+    cy.get('[data-testid=DatabaseAppObject_Chinook]').rightclick();
+    cy.contains('Chinook').rightclick();
+    cy.contains('Restore/import SQL dump').click();
+    cy.get('#uploadFileButton').selectFile('data/chinook-mysql.sql', { force: true });
+    cy.wait(500);
+    cy.get('[data-testid=ImportDatabaseDumpModal_runImport]').click();
+    cy.contains('Importing database');
+    cy.contains('Finished job script');
+    cy.get('[data-testid=RunScriptModal_close]').click();
+    cy.contains('Chinook').click();
+    cy.contains('Album');
+  });
+
+  it('Import Chinook Postgres', () => {
+    cy.visit('http://localhost:3000');
+    cy.contains('Postgres-connection').click();
+    cy.get('[data-testid=DatabaseAppObject_Chinook]').rightclick();
+    cy.contains('Restore/import SQL dump').click();
+    cy.get('#uploadFileButton').selectFile('data/chinook-postgres.sql', { force: true });
+    cy.wait(500);
+    cy.get('[data-testid=ImportDatabaseDumpModal_runImport]').click();
+    cy.contains('Importing database');
+    cy.contains('Finished job script');
+    cy.get('[data-testid=RunScriptModal_close]').click();
+    cy.contains('Chinook').click();
+    cy.contains('album');
+  });
+
+  it('Open ask pwd connection', () => {
+    cy.visit('http://localhost:3000');
+    cy.contains('Postgres-ask-connection').click();
+    cy.testid('DatabaseLoginModal_username').clear().type('postgres');
+    cy.testid('DatabaseLoginModal_password').clear().type('Pwd2020Db');
+    cy.testid('DatabaseLoginModal_connect').click();
+    cy.contains('Chinook').click();
+    cy.contains('album');
+  });
+
+  // it('import chinook DB', () => {
+  //   cy.visit('http://localhost:3000');
+  //   cy.get('[data-testid=ConnectionTab_buttonConnect]').click();
+  // });
+});

e2e-tests/cypress/fixtures/example.json

@@ -0,0 +1,5 @@
+{
+  "name": "Using fixtures to represent data",
+  "email": "hello@cypress.io",
+  "body": "Fixtures are a great way to mock data for responses to routes"
+}

e2e-tests/cypress/support/commands.js

@@ -0,0 +1,29 @@
+// ***********************************************
+// This example commands.js shows you how to
+// create various custom commands and overwrite
+// existing commands.
+//
+// For more comprehensive examples of custom
+// commands please read more here:
+// https://on.cypress.io/custom-commands
+// ***********************************************
+//
+//
+// -- This is a parent command --
+// Cypress.Commands.add('login', (email, password) => { ... })
+//
+//
+// -- This is a child command --
+// Cypress.Commands.add('drag', { prevSubject: 'element'}, (subject, options) => { ... })
+//
+//
+// -- This is a dual command --
+// Cypress.Commands.add('dismiss', { prevSubject: 'optional'}, (subject, options) => { ... })
+//
+//
+// -- This will overwrite an existing command --
+// Cypress.Commands.overwrite('visit', (originalFn, url, options) => { ... })
+
+Cypress.Commands.add('testid', (testId, options = {}) => {
+  return cy.get(`[data-testid="${testId}"]`, options);
+});

e2e-tests/cypress/support/e2e.js

@@ -0,0 +1,44 @@
+// ***********************************************************
+// This example support/e2e.js is processed and
+// loaded automatically before your test files.
+//
+// This is a great place to put global configuration and
+// behavior that modifies Cypress.
+//
+// You can change the location of this file or turn off
+// automatically serving support files with the
+// 'supportFile' configuration option.
+//
+// You can read more here:
+// https://on.cypress.io/configuration
+// ***********************************************************
+
+// Import commands.js using ES2015 syntax:
+import './commands';
+
+// Alternatively you can use CommonJS syntax:
+// require('./commands')
+
+import 'cypress-real-events';
+
+beforeEach(() => {
+  // Replace 'my-database-name' with the actual IndexedDB name
+  cy.window().then(win => {
+    return new Promise((resolve, reject) => {
+      const request = win.indexedDB.deleteDatabase('localforage');
+      request.onsuccess = () => {
+        // Database successfully deleted
+        resolve();
+      };
+      request.onerror = () => {
+        // Some error occurred
+        reject(request.error);
+      };
+      request.onblocked = () => {
+        // Might happen if there are open connections
+        console.warn('IndexedDB deletion blocked');
+        resolve();
+      };
+    });
+  });
+});

e2e-tests/docker-compose.yaml

@@ -0,0 +1,37 @@
+version: '3'
+services:
+  postgres:
+    image: postgres
+    restart: always
+    environment:
+      POSTGRES_PASSWORD: Pwd2020Db
+    ports: 
+      - 16000:5432
+
+  mariadb:
+    image: mariadb
+    command: --default-authentication-plugin=mysql_native_password
+    restart: always
+    ports: 
+      - 16004:3306
+    environment:
+      - MYSQL_ROOT_PASSWORD=Pwd2020Db
+
+  mysql-ssh-login:
+    build: containers/mysql-ssh-login
+    restart: always
+    ports: 
+      - 16005:3306
+      - "16006:22"
+
+  mysql-ssh-keyfile:
+    build: containers/mysql-ssh-keyfile
+    restart: always
+    ports: 
+      - 16007:3306
+      - "16008:22"
+
+  dex:
+    build: containers/dex
+    ports:
+      - "16009:5556"

e2e-tests/e2eTestTools.js

@@ -0,0 +1,29 @@
+const path = require('path');
+const os = require('os');
+const fs = require('fs');
+
+const baseDir = path.join(os.homedir(), '.dbgate');
+
+// function createTimeStamp() {
+//   const now = new Date();
+//   const year = now.getFullYear();
+//   const month = String(now.getMonth() + 1).padStart(2, '0'); // měsíc je 0-indexovaný
+//   const day = String(now.getDate()).padStart(2, '0');
+//   const hours = String(now.getHours()).padStart(2, '0');
+//   const minutes = String(now.getMinutes()).padStart(2, '0');
+//   const seconds = String(now.getSeconds()).padStart(2, '0');
+
+//   // Poskládáme datum a čas do názvu souboru
+//   const ts = `${year}-${month}-${day}_${hours}-${minutes}-${seconds}`;
+//   return ts;
+// }
+
+function clearTestingData() {
+  if (fs.existsSync(path.join(baseDir, 'connections-e2etests.jsonl'))) {
+    fs.unlinkSync(path.join(baseDir, 'connections-e2etests.jsonl'));
+  }
+}
+
+module.exports = {
+  clearTestingData,
+};

e2e-tests/env/browse-data/.env

@@ -0,0 +1,8 @@
+CONNECTIONS=mysql
+
+LABEL_mysql=MySql-connection
+SERVER_mysql=localhost
+USER_mysql=root
+PASSWORD_mysql=Pwd2020Db
+PORT_mysql=16004
+ENGINE_mysql=mysql@dbgate-plugin-mysql

e2e-tests/env/oauth/.env

@@ -0,0 +1,15 @@
+CONNECTIONS=mysql
+
+LABEL_mysql=MySql-connection
+SERVER_mysql=localhost
+USER_mysql=root
+PASSWORD_mysql=Pwd2020Db
+PORT_mysql=16004
+ENGINE_mysql=mysql@dbgate-plugin-mysql
+
+OAUTH_AUTH=http://localhost:16009/dex/auth
+OAUTH_TOKEN=http://localhost:16009/dex/token
+OAUTH_CLIENT_ID=my-app
+OAUTH_CLIENT_SECRET=my-secret
+OAUTH_LOGIN_FIELD=username
+OAUTH_SCOPE=openid

e2e-tests/env/portal/.env

@@ -0,0 +1,22 @@
+CONNECTIONS=mysql,postgres,pgask
+
+LABEL_mysql=MySql-connection
+SERVER_mysql=localhost
+USER_mysql=root
+PASSWORD_mysql=Pwd2020Db
+PORT_mysql=16004
+ENGINE_mysql=mysql@dbgate-plugin-mysql
+
+LABEL_postgres=Postgres-connection
+SERVER_postgres=localhost
+USER_postgres=postgres
+PASSWORD_postgres=Pwd2020Db
+PORT_postgres=16000
+ENGINE_postgres=postgres@dbgate-plugin-postgres
+
+LABEL_pgask=Postgres-ask-connection
+SERVER_pgask=localhost
+USER_pgask=postgres
+PORT_pgask=16000
+ENGINE_pgask=postgres@dbgate-plugin-postgres
+PASSWORD_MODE_pgask=askUser

e2e-tests/init/browse-data.js

@@ -0,0 +1,43 @@
+const dbgateApi = require('dbgate-api');
+dbgateApi.initializeApiEnvironment();
+const dbgatePluginMysql = require('dbgate-plugin-mysql');
+dbgateApi.registerPlugins(dbgatePluginMysql);
+const dbgatePluginPostgres = require('dbgate-plugin-postgres');
+dbgateApi.registerPlugins(dbgatePluginPostgres);
+
+async function run() {
+  await dbgateApi.executeQuery({
+    connection: {
+      server: process.env.SERVER_mysql,
+      user: process.env.USER_mysql,
+      password: process.env.PASSWORD_mysql,
+      port: process.env.PORT_mysql,
+      engine: 'mysql@dbgate-plugin-mysql',
+    },
+    sql: 'drop database if exists Chinook',
+  });
+
+  await dbgateApi.executeQuery({
+    connection: {
+      server: process.env.SERVER_mysql,
+      user: process.env.USER_mysql,
+      password: process.env.PASSWORD_mysql,
+      port: process.env.PORT_mysql,
+      engine: 'mysql@dbgate-plugin-mysql',
+    },
+    sql: 'create database Chinook',
+  });
+
+  await dbgateApi.importDatabase({
+    connection: {
+      server: process.env.SERVER_mysql,
+      user: process.env.USER_mysql,
+      password: process.env.PASSWORD_mysql,
+      port: process.env.PORT_mysql,
+      engine: 'mysql@dbgate-plugin-mysql',
+    },
+    inputFile: '../data/Chinook-mysql.sql',
+  });
+}
+
+dbgateApi.runScript(run);

e2e-tests/init/portal.js

@@ -0,0 +1,32 @@
+const dbgateApi = require('dbgate-api');
+dbgateApi.initializeApiEnvironment();
+const dbgatePluginMysql = require('dbgate-plugin-mysql');
+dbgateApi.registerPlugins(dbgatePluginMysql);
+const dbgatePluginPostgres = require('dbgate-plugin-postgres');
+dbgateApi.registerPlugins(dbgatePluginPostgres);
+
+async function run() {
+    await dbgateApi.executeQuery({
+      connection: {
+        server: process.env.SERVER_mysql,
+        user: process.env.USER_mysql,
+        password: process.env.PASSWORD_mysql,
+        port: process.env.PORT_mysql,
+        engine: 'mysql@dbgate-plugin-mysql',
+      },
+      sql: 'drop database if exists Chinook',
+    });
+
+  await dbgateApi.executeQuery({
+    connection: {
+      server: process.env.SERVER_postgres,
+      user: process.env.USER_postgres,
+      password: process.env.PASSWORD_postgres,
+      port: process.env.PORT_postgres,
+      engine: 'postgres@dbgate-plugin-postgres',
+    },
+    sql: 'drop database if exists "Chinook"',
+  });
+}
+
+dbgateApi.runScript(run);

e2e-tests/package.json

@@ -0,0 +1,31 @@
+{
+  "name": "e2e-tests",
+  "version": "1.0.0",
+  "main": "index.js",
+  "license": "GPL",
+  "devDependencies": {
+    "axios": "^1.7.9",
+    "cross-env": "^7.0.3",
+    "cypress": "^13.16.1",
+    "cypress-real-events": "^1.13.0",
+    "env-cmd": "^10.1.0",
+    "kill-port": "^2.0.1",
+    "start-server-and-test": "^2.0.8"
+  },
+  "scripts": {
+    "cy:open": "cypress open --config experimentalInteractiveRunEvents=true",
+
+    "cy:run:add-connection": "cypress run --spec cypress/e2e/add-connection.cy.js",
+    "cy:run:portal": "cypress run --spec cypress/e2e/portal.cy.js",
+    "cy:run:oauth": "cypress run --spec cypress/e2e/oauth.cy.js",
+    "cy:run:browse-data": "cypress run --spec cypress/e2e/browse-data.cy.js",
+
+    "start:add-connection": "cd .. && node packer/build/bundle.js --listen-api --run-e2e-tests",
+    "start:portal": "cd .. && env-cmd -f e2e-tests/env/portal/.env node e2e-tests/init/portal.js && env-cmd -f e2e-tests/env/portal/.env node packer/build/bundle.js --listen-api --run-e2e-tests",
+    "start:oauth": "cd .. && env-cmd -f e2e-tests/env/oauth/.env node packer/build/bundle.js --listen-api --run-e2e-tests",
+    "start:browse-data": "cd .. && env-cmd -f e2e-tests/env/browse-data/.env node packer/build/bundle.js --listen-api --run-e2e-tests",
+
+    "test": "start-server-and-test start:add-connection http://localhost:3000 cy:run:add-connection && start-server-and-test start:portal http://localhost:3000 cy:run:portal && start-server-and-test start:oauth http://localhost:3000 cy:run:oauth && start-server-and-test start:browse-data http://localhost:3000 cy:run:browse-data",
+    "test:ci": "yarn test"
+  }
+}

integration-tests/__tests__/alter-database.spec.js

@@ -3,9 +3,15 @@ const _ = require('lodash');
 const fp = require('lodash/fp');
 const { testWrapper } = require('../tools');
 const engines = require('../engines');
-const { getAlterDatabaseScript, extendDatabaseInfo, generateDbPairingId } = require('dbgate-tools');
+const {
+  getAlterDatabaseScript,
+  extendDatabaseInfo,
+  generateDbPairingId,
+  formatQueryWithoutParams,
+  runCommandOnDriver,
+} = require('dbgate-tools');
 
-const initSql = ['CREATE TABLE t1 (id int primary key)', 'CREATE TABLE t2 (id int primary key)'];
+const initSql = ['CREATE TABLE ~t1 (~id int primary key)', 'CREATE TABLE ~t2 (~id int primary key)'];
 
 function flatSource(engineCond = x => !x.skipReferences) {
   return _.flatten(
@@ -16,13 +22,14 @@ function flatSource(engineCond = x => !x.skipReferences) {
 }
 
 async function testDatabaseDiff(conn, driver, mangle, createObject = null) {
-  await driver.query(conn, `create table t1 (id int not null primary key)`);
+  await runCommandOnDriver(conn, driver, `create table ~t1 (~id int not null primary key)`);
 
-  await driver.query(
+  await runCommandOnDriver(
     conn,
-    `create table t2 (
-    id int not null primary key, 
-    t1_id int null references t1(id)
+    driver,
+    `create table ~t2 (
+    ~id int not null primary key, 
+    ~t1_id int null references ~t1(~id)
   )`
   );
 
@@ -63,29 +70,32 @@ describe('Alter database', () => {
         db => {
           _.remove(db[type], x => x.pureName == 'obj1');
         },
-        object.create1
+        formatQueryWithoutParams(driver, object.create1)
       );
       expect(db[type].length).toEqual(0);
     })
   );
 
-  test.each(flatSource(x => x.supportRenameSqlObject))(
-    'Rename object - %s - %s',
-    testWrapper(async (conn, driver, type, object, engine) => {
-      for (const sql of initSql) await driver.query(conn, sql, { discardResult: true });
+  const objectsSupportingRename = flatSource(x => x.supportRenameSqlObject);
+  if (objectsSupportingRename.length > 0) {
+    test.each(objectsSupportingRename)(
+      'Rename object - %s - %s',
+      testWrapper(async (conn, driver, type, object, engine) => {
+        for (const sql of initSql) await runCommandOnDriver(conn, driver, sql);
 
-      await driver.query(conn, object.create1, { discardResult: true });
+        await runCommandOnDriver(conn, driver, object.create1);
 
-      const structure = extendDatabaseInfo(await driver.analyseFull(conn));
+        const structure = extendDatabaseInfo(await driver.analyseFull(conn));
 
-      const dmp = driver.createDumper();
-      dmp.renameSqlObject(structure[type][0], 'renamed1');
+        const dmp = driver.createDumper();
+        dmp.renameSqlObject(structure[type][0], 'renamed1');
 
-      await driver.query(conn, dmp.s);
+        await driver.query(conn, dmp.s);
 
-      const structure2 = await driver.analyseFull(conn);
-      expect(structure2[type].length).toEqual(1);
-      expect(structure2[type][0].pureName).toEqual('renamed1');
-    })
-  );
+        const structure2 = await driver.analyseFull(conn);
+        expect(structure2[type].length).toEqual(1);
+        expect(structure2[type][0].pureName).toEqual('renamed1');
+      })
+    );
+  }
 });

integration-tests/__tests__/alter-table.spec.js

@@ -4,7 +4,12 @@ const fp = require('lodash/fp');
 const { testWrapper } = require('../tools');
 const engines = require('../engines');
 const crypto = require('crypto');
-const { getAlterTableScript, extendDatabaseInfo, generateDbPairingId } = require('dbgate-tools');
+const {
+  getAlterTableScript,
+  extendDatabaseInfo,
+  generateDbPairingId,
+  formatQueryWithoutParams,
+} = require('dbgate-tools');
 
 function pickImportantTableInfo(engine, table) {
   const props = ['columnName', 'defaultValue'];
@@ -15,7 +20,10 @@ function pickImportantTableInfo(engine, table) {
     columns: table.columns
       .filter(x => x.columnName != 'rowid')
       .map(fp.pick(props))
-      .map(props => _.omitBy(props, x => x == null)),
+      .map(props => _.omitBy(props, x => x == null))
+      .map(props =>
+        _.omitBy(props, (v, k) => k == 'defaultValue' && v == 'NULL' && engine.setNullDefaultInsteadOfDrop)
+      ),
   };
 }
 
@@ -25,27 +33,36 @@ function checkTableStructure(engine, t1, t2) {
 }
 
 async function testTableDiff(engine, conn, driver, mangle) {
-  await driver.query(conn, `create table t0 (id int not null primary key)`);
+  await driver.query(conn, formatQueryWithoutParams(driver, `create table ~t0 (~id int not null primary key)`));
 
   await driver.query(
     conn,
-    `create table t1 (
-    col_pk int not null primary key, 
-    col_std int, 
-    col_def int default 12,
-    ${engine.skipReferences ? '' : 'col_fk int references t0(id),'}
-    col_idx int,
-    col_uq int ${engine.skipUnique ? '' : 'unique'} ,
-    col_ref int ${engine.skipUnique ? '' : 'unique'}
+    formatQueryWithoutParams(
+      driver,
+      `create table ~t1 (
+    ~col_pk int not null primary key, 
+    ~col_std int, 
+    ~col_def int default 12,
+    ${engine.skipReferences ? '' : '~col_fk int references ~t0(~id),'}
+    ~col_idx int,
+    ~col_uq int ${engine.skipUnique ? '' : 'unique'} ,
+    ~col_ref int ${engine.skipUnique ? '' : 'unique'}
   )`
+    )
   );
 
   if (!engine.skipIndexes) {
-    await driver.query(conn, `create index idx1 on t1(col_idx)`);
+    await driver.query(conn, formatQueryWithoutParams(driver, `create index ~idx1 on ~t1(~col_idx)`));
   }
 
   if (!engine.skipReferences) {
-    await driver.query(conn, `create table t2 (id int not null primary key, fkval int null references t1(col_ref))`);
+    await driver.query(
+      conn,
+      formatQueryWithoutParams(
+        driver,
+        `create table ~t2 (~id int not null primary key, ~fkval int null references ~t1(~col_ref))`
+      )
+    );
   }
 
   const tget = x => x.tables.find(y => y.pureName == 't1');
@@ -175,5 +192,4 @@ describe('Alter table', () => {
   //     });
   //   })
   // );
-
 });

integration-tests/__tests__/data-duplicator.spec.js

@@ -2,7 +2,7 @@ const engines = require('../engines');
 const stream = require('stream');
 const { testWrapper } = require('../tools');
 const dataDuplicator = require('dbgate-api/src/shell/dataDuplicator');
-const { runCommandOnDriver } = require('dbgate-tools');
+const { runCommandOnDriver, runQueryOnDriver } = require('dbgate-tools');
 
 describe('Data duplicator', () => {
   test.each(engines.filter(x => !x.skipDataDuplicator).map(engine => [engine.label, engine]))(
@@ -84,10 +84,10 @@ describe('Data duplicator', () => {
         ],
       });
 
-      const res1 = await driver.query(conn, `select count(*) as cnt from t1`);
+      const res1 = await runQueryOnDriver(conn, driver, dmp => dmp.put(`select count(*) as ~cnt from ~t1`));
       expect(res1.rows[0].cnt.toString()).toEqual('6');
 
-      const res2 = await driver.query(conn, `select count(*) as cnt from t2`);
+      const res2 = await runQueryOnDriver(conn, driver, dmp => dmp.put(`select count(*) as ~cnt from ~t2`));
       expect(res2.rows[0].cnt.toString()).toEqual('6');
     })
   );
@@ -145,13 +145,15 @@ describe('Data duplicator', () => {
         },
       });
 
-      const res1 = await driver.query(conn, `select count(*) as cnt from t1`);
+      const res1 = await runQueryOnDriver(conn, driver, dmp => dmp.put(`select count(*) as ~cnt from ~t1`));
       expect(res1.rows[0].cnt.toString()).toEqual('1');
 
-      const res2 = await driver.query(conn, `select count(*) as cnt from t2`);
+      const res2 = await runQueryOnDriver(conn, driver, dmp => dmp.put(`select count(*) as ~cnt from ~t2`));
       expect(res2.rows[0].cnt.toString()).toEqual('2');
 
-      const res3 = await driver.query(conn, `select count(*) as cnt from t2 where valfk is not null`);
+      const res3 = await runQueryOnDriver(conn, driver, dmp =>
+        dmp.put(`select count(*) as ~cnt from ~t2 where ~valfk is not null`)
+      );
       expect(res3.rows[0].cnt.toString()).toEqual('1');
     })
   );

integration-tests/__tests__/db-import-export.spec.js

@@ -2,9 +2,11 @@ const engines = require('../engines');
 const stream = require('stream');
 const { testWrapper } = require('../tools');
 const tableWriter = require('dbgate-api/src/shell/tableWriter');
+const tableReader = require('dbgate-api/src/shell/tableReader');
 const copyStream = require('dbgate-api/src/shell/copyStream');
 const importDatabase = require('dbgate-api/src/shell/importDatabase');
 const fakeObjectReader = require('dbgate-api/src/shell/fakeObjectReader');
+const { runQueryOnDriver, runCommandOnDriver } = require('dbgate-tools');
 
 function createImportStream() {
   const pass = new stream.PassThrough({
@@ -22,7 +24,17 @@ function createImportStream() {
   return pass;
 }
 
-describe('DB Import', () => {
+function createExportStream() {
+  const writable = new stream.Writable({ objectMode: true });
+  writable.resultArray = [];
+  writable._write = (chunk, encoding, callback) => {
+    writable.resultArray.push(chunk);
+    callback();
+  };
+  return writable;
+}
+
+describe('DB Import/export', () => {
   test.each(engines.map(engine => [engine.label, engine]))(
     'Import one table - %s',
     testWrapper(async (conn, driver, engine) => {
@@ -37,7 +49,7 @@ describe('DB Import', () => {
       });
       await copyStream(reader, writer);
 
-      const res = await driver.query(conn, `select count(*) as cnt from t1`);
+      const res = await runQueryOnDriver(conn, driver, dmp => dmp.put(`select count(*) as ~cnt from ~t1`));
       expect(res.rows[0].cnt.toString()).toEqual('6');
     })
   );
@@ -65,10 +77,10 @@ describe('DB Import', () => {
       });
       await copyStream(reader2, writer2);
 
-      const res1 = await driver.query(conn, `select count(*) as cnt from t1`);
+      const res1 = await runQueryOnDriver(conn, driver, dmp => dmp.put(`select count(*) as ~cnt from ~t1`));
       expect(res1.rows[0].cnt.toString()).toEqual('6');
 
-      const res2 = await driver.query(conn, `select count(*) as cnt from t2`);
+      const res2 = await runQueryOnDriver(conn, driver, dmp => dmp.put(`select count(*) as ~cnt from ~t2`));
       expect(res2.rows[0].cnt.toString()).toEqual('6');
     })
   );
@@ -98,4 +110,35 @@ describe('DB Import', () => {
       // expect(res2.rows[0].cnt.toString()).toEqual('6');
     })
   );
+
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Export one table - %s',
+    testWrapper(async (conn, driver, engine) => {
+      // const reader = await fakeObjectReader({ delay: 10 });
+      // const reader = await fakeObjectReader();
+      await runCommandOnDriver(conn, driver, 'create table ~t1 (~id int primary key, ~country varchar(100))');
+      const data = [
+        [1, 'Czechia'],
+        [2, 'Austria'],
+        [3, 'Germany'],
+        [4, 'Romania'],
+        [5, 'Great Britain'],
+        [6, 'Bosna, Hecegovina'],
+      ];
+      for (const row of data) {
+        await runCommandOnDriver(conn, driver, dmp =>
+          dmp.put('insert into ~t1(~id, ~country) values (%v, %v)', ...row)
+        );
+      }
+      const reader = await tableReader({
+        systemConnection: conn,
+        driver,
+        pureName: 't1',
+      });
+      const writer = createExportStream();
+      await copyStream(reader, writer);
+
+      expect(writer.resultArray.filter(x => !x.__isStreamHeader).map(row => [row.id, row.country])).toEqual(data);
+    })
+  );
 });

integration-tests/__tests__/deploy-database.spec.js

@@ -4,7 +4,7 @@ const { testWrapper, testWrapperPrepareOnly } = require('../tools');
 const _ = require('lodash');
 const engines = require('../engines');
 const deployDb = require('dbgate-api/src/shell/deployDb');
-const { databaseInfoFromYamlModel } = require('dbgate-tools');
+const { databaseInfoFromYamlModel, runQueryOnDriver, formatQueryWithoutParams } = require('dbgate-tools');
 const generateDeploySql = require('dbgate-api/src/shell/generateDeploySql');
 const connectUtility = require('dbgate-api/src/utility/connectUtility');
 
@@ -69,6 +69,29 @@ function checkStructure(
   }
 }
 
+// function convertObjectText(text, driver) {
+//   if (!text) return undefined;
+//   text = formatQueryWithoutParams(driver, text);
+//   if (driver.dialect.requireFromDual && text.startsWith('create view ') && !text.includes('from')) {
+//     text = text + ' from dual';
+//   }
+//   return text;
+// }
+
+// function convertModelToEngine(model, driver) {
+//   return model.map(x => ({
+//     ...x,
+//     text: convertObjectText(x.text, driver),
+//   }));
+// }
+
+function convertModelToEngine(model, driver) {
+  return model.map(x => ({
+    ...x,
+    text: x.text ? formatQueryWithoutParams(driver, x.text) : undefined,
+  }));
+}
+
 async function testDatabaseDeploy(engine, conn, driver, dbModelsYaml, options) {
   const { testEmptyLastScript, finalCheckAgainstModel, markDeleted, allowDropStatements } = options || {};
   let index = 0;
@@ -83,13 +106,13 @@ async function testDatabaseDeploy(engine, conn, driver, dbModelsYaml, options) {
 
   for (const loadedDbModel of dbModelsYaml) {
     if (_.isString(loadedDbModel)) {
-      await driver.script(conn, loadedDbModel);
+      await driver.script(conn, formatQueryWithoutParams(driver, loadedDbModel));
     } else {
       const { sql, isEmpty } = await generateDeploySql({
         systemConnection: conn.isPreparedOnly ? undefined : conn,
         connection: conn.isPreparedOnly ? conn : undefined,
         driver,
-        loadedDbModel,
+        loadedDbModel: convertModelToEngine(loadedDbModel, driver),
         dbdiffOptionsExtra,
       });
       console.debug('Generated deploy script:', sql);
@@ -106,7 +129,7 @@ async function testDatabaseDeploy(engine, conn, driver, dbModelsYaml, options) {
         systemConnection: conn.isPreparedOnly ? undefined : conn,
         connection: conn.isPreparedOnly ? conn : undefined,
         driver,
-        loadedDbModel,
+        loadedDbModel: convertModelToEngine(loadedDbModel, driver),
         dbdiffOptionsExtra,
       });
     }
@@ -117,7 +140,12 @@ async function testDatabaseDeploy(engine, conn, driver, dbModelsYaml, options) {
   const dbhan = conn.isPreparedOnly ? await connectUtility(driver, conn, 'read') : conn;
   const structure = await driver.analyseFull(dbhan);
   if (conn.isPreparedOnly) await driver.close(dbhan);
-  checkStructure(engine, structure, finalCheckAgainstModel ?? _.findLast(dbModelsYaml, x => _.isArray(x)), options);
+  checkStructure(
+    engine,
+    structure,
+    convertModelToEngine(finalCheckAgainstModel ?? _.findLast(dbModelsYaml, x => _.isArray(x)), driver),
+    options
+  );
 }
 
 describe('Deploy database', () => {
@@ -339,7 +367,7 @@ describe('Deploy database', () => {
         ],
       ]);
 
-      const res = await driver.query(conn, `select count(*) as cnt from t1`);
+      const res = await runQueryOnDriver(conn, driver, `select count(*) as ~cnt from ~t1`);
       expect(res.rows[0].cnt.toString()).toEqual('3');
     })
   );
@@ -386,12 +414,12 @@ describe('Deploy database', () => {
         ],
       ]);
 
-      const res = await driver.query(conn, `select val from t1 where id = 2`);
+      const res = await runQueryOnDriver(conn, driver, `select ~val from ~t1 where ~id = 2`);
       expect(res.rows[0].val.toString()).toEqual('5');
     })
   );
 
-  test.each(engines.enginesPostgre.map(engine => [engine.label, engine]))(
+  test.each([engines.postgreSqlEngine].map(engine => [engine.label, engine]))(
     'Current timestamp default value - %s',
     testWrapper(async (conn, driver, engine) => {
       await testDatabaseDeploy(engine, conn, driver, [
@@ -414,8 +442,8 @@ describe('Deploy database', () => {
         ],
       ]);
 
-      await driver.query(conn, `insert into t1 (id) values (1)`);
-      const res = await driver.query(conn, ` select val from t1 where id = 1`);
+      await runQueryOnDriver(conn, driver, `insert into ~t1 (~id) values (1)`);
+      const res = await runQueryOnDriver(conn, driver, ` select ~val from ~t1 where ~id = 1`);
       expect(res.rows[0].val.toString().substring(0, 2)).toEqual('20');
     })
   );
@@ -438,7 +466,7 @@ describe('Deploy database', () => {
             },
           },
         ],
-        'insert into t1 (id, val) values (1, 1); insert into t1 (id) values (2)',
+        'insert into ~t1 (~id, ~val) values (1, 1); insert into ~t1 (~id) values (2)',
         [
           {
             name: 't1.table.yaml',
@@ -452,16 +480,16 @@ describe('Deploy database', () => {
             },
           },
         ],
-        'insert into t1 (id) values (3);',
+        'insert into ~t1 (~id) values (3);',
       ]);
 
-      const res1 = await driver.query(conn, `select val from t1 where id = 1`);
+      const res1 = await runQueryOnDriver(conn, driver, `select ~val from ~t1 where ~id = 1`);
       expect(res1.rows[0].val).toEqual(1);
 
-      const res2 = await driver.query(conn, `select val from t1 where id = 2`);
+      const res2 = await runQueryOnDriver(conn, driver, `select ~val from ~t1 where ~id = 2`);
       expect(res2.rows[0].val).toEqual(20);
 
-      const res3 = await driver.query(conn, `select val from t1 where id = 3`);
+      const res3 = await runQueryOnDriver(conn, driver, `select ~val from ~t1 where ~id = 3`);
       expect(res2.rows[0].val).toEqual(20);
     })
   );
@@ -525,17 +553,17 @@ describe('Deploy database', () => {
 
   const V1 = {
     name: 'v1.view.sql',
-    text: 'create view v1 as select * from t1',
+    text: 'create view ~v1 as select * from ~t1',
   };
 
   const V1_VARIANT2 = {
     name: 'v1.view.sql',
-    text: 'create view v1 as select 1 as c1',
+    text: 'create view ~v1 as select ~id + ~id ~idsum from ~t1',
   };
 
   const V1_DELETED = {
     name: '_deleted_v1.view.sql',
-    text: 'create view _deleted_v1 as select * from t1',
+    text: 'create view ~_deleted_v1 as select * from ~t1',
   };
 
   test.each(engines.map(engine => [engine.label, engine]))(
@@ -682,15 +710,15 @@ describe('Deploy database', () => {
         [
           {
             name: '1.predeploy.sql',
-            text: 'create table t1 (id int primary key); insert into t1 (id) values (1);',
+            text: 'create table ~t1 (~id int primary key); insert into ~t1 (~id) values (1);',
           },
         ],
       ]);
 
-      const res1 = await driver.query(conn, 'SELECT COUNT(*) AS cnt FROM t1');
+      const res1 = await runQueryOnDriver(conn, driver, 'SELECT COUNT(*) AS ~cnt FROM ~t1');
       expect(res1.rows[0].cnt == 1).toBeTruthy();
 
-      const res2 = await driver.query(conn, 'SELECT COUNT(*) AS cnt FROM dbgate_deploy_journal');
+      const res2 = await runQueryOnDriver(conn, driver, 'SELECT COUNT(*) AS ~cnt FROM ~dbgate_deploy_journal');
       expect(res2.rows[0].cnt == 1).toBeTruthy();
     })
   );
@@ -702,48 +730,53 @@ describe('Deploy database', () => {
         [
           {
             name: 't1.uninstall.sql',
-            text: 'drop table t1',
+            text: 'drop table ~t1',
           },
           {
             name: 't1.install.sql',
-            text: 'create table t1 (id int primary key); insert into t1 (id) values (1)',
+            text: 'create table ~t1 (~id int primary key); insert into ~t1 (~id) values (1)',
           },
           {
             name: 't2.once.sql',
-            text: 'create table t2 (id int primary key); insert into t2 (id) values (1)',
+            text: 'create table ~t2 (~id int primary key); insert into ~t2 (~id) values (1)',
           },
         ],
         [
           {
             name: 't1.uninstall.sql',
-            text: 'drop table t1',
+            text: 'drop table ~t1',
           },
           {
             name: 't1.install.sql',
-            text: 'create table t1 (id int primary key, val int); insert into t1 (id, val) values (1, 11)',
+            text: 'create table ~t1 (~id int primary key, ~val int); insert into ~t1 (~id, ~val) values (1, 11)',
           },
           {
             name: 't2.once.sql',
-            text: 'insert into t2 (id) values (2)',
+            text: 'insert into ~t2 (~id) values (2)',
           },
         ],
       ]);
 
-      const res1 = await driver.query(conn, 'SELECT val from t1 where id = 1');
+      const res1 = await runQueryOnDriver(conn, driver, 'SELECT ~val from ~t1 where ~id = 1');
       expect(res1.rows[0].val == 11).toBeTruthy();
 
-      const res2 = await driver.query(conn, 'SELECT COUNT(*) AS cnt FROM t2');
+      const res2 = await runQueryOnDriver(conn, driver, 'SELECT COUNT(*) AS ~cnt FROM ~t2');
       expect(res2.rows[0].cnt == 1).toBeTruthy();
 
-      const res3 = await driver.query(conn, 'SELECT COUNT(*) AS cnt FROM dbgate_deploy_journal');
+      const res3 = await runQueryOnDriver(conn, driver, 'SELECT COUNT(*) AS ~cnt FROM ~dbgate_deploy_journal');
       expect(res3.rows[0].cnt == 3).toBeTruthy();
 
-      const res4 = await driver.query(conn, "SELECT run_count from dbgate_deploy_journal where name = 't2.once.sql'");
+      const res4 = await runQueryOnDriver(
+        conn,
+        driver,
+        "SELECT ~run_count from ~dbgate_deploy_journal where ~name = 't2.once.sql'"
+      );
       expect(res4.rows[0].run_count == 1).toBeTruthy();
 
-      const res5 = await driver.query(
+      const res5 = await runQueryOnDriver(
         conn,
-        "SELECT run_count from dbgate_deploy_journal where name = 't1.install.sql'"
+        driver,
+        "SELECT ~run_count from ~dbgate_deploy_journal where ~name = 't1.install.sql'"
       );
       expect(res5.rows[0].run_count == 2).toBeTruthy();
     })

integration-tests/__tests__/object-analyse.spec.js

@@ -1,8 +1,9 @@
 const { testWrapper } = require('../tools');
 const engines = require('../engines');
 const _ = require('lodash');
+const { formatQueryWithoutParams, runCommandOnDriver } = require('dbgate-tools');
 
-const initSql = ['CREATE TABLE t1 (id int primary key)', 'CREATE TABLE t2 (id int primary key)'];
+const initSql = ['CREATE TABLE ~t1 (~id int primary key)', 'CREATE TABLE ~t2 (~id int primary key)'];
 
 function flatSource() {
   return _.flatten(
@@ -18,6 +19,16 @@ function flatSourceParameters() {
   );
 }
 
+function flatSourceTriggers() {
+  return _.flatten(engines.map(engine => (engine.triggers || []).map(trigger => [engine.label, trigger, engine])));
+}
+
+function flatSourceSchedulerEvents() {
+  return _.flatten(
+    engines.map(engine => (engine.schedulerEvents || []).map(schedulerEvent => [engine.label, schedulerEvent, engine]))
+  );
+}
+
 const obj1Match = expect.objectContaining({
   pureName: 'obj1',
 });
@@ -34,9 +45,9 @@ describe('Object analyse', () => {
   test.each(flatSource())(
     'Full analysis - %s - %s',
     testWrapper(async (conn, driver, type, object, engine) => {
-      for (const sql of initSql) await driver.query(conn, sql, { discardResult: true });
+      for (const sql of initSql) await runCommandOnDriver(conn, driver, sql);
 
-      await driver.query(conn, object.create1, { discardResult: true });
+      await runCommandOnDriver(conn, driver, object.create1);
       const structure = await driver.analyseFull(conn);
 
       expect(structure[type].length).toEqual(1);
@@ -47,11 +58,11 @@ describe('Object analyse', () => {
   test.each(flatSource())(
     'Incremental analysis - add - %s - %s',
     testWrapper(async (conn, driver, type, object, engine) => {
-      for (const sql of initSql) await driver.query(conn, sql, { discardResult: true });
+      for (const sql of initSql) await runCommandOnDriver(conn, driver, sql);
 
-      await driver.query(conn, object.create2, { discardResult: true });
+      await runCommandOnDriver(conn, driver, object.create2);
       const structure1 = await driver.analyseFull(conn);
-      await driver.query(conn, object.create1, { discardResult: true });
+      await runCommandOnDriver(conn, driver, object.create1);
       const structure2 = await driver.analyseIncremental(conn, structure1);
 
       expect(structure2[type].length).toEqual(2);
@@ -62,12 +73,12 @@ describe('Object analyse', () => {
   test.each(flatSource())(
     'Incremental analysis - drop - %s - %s',
     testWrapper(async (conn, driver, type, object, engine) => {
-      for (const sql of initSql) await driver.query(conn, sql, { discardResult: true });
+      for (const sql of initSql) await runCommandOnDriver(conn, driver, sql);
 
-      await driver.query(conn, object.create1, { discardResult: true });
-      await driver.query(conn, object.create2, { discardResult: true });
+      await runCommandOnDriver(conn, driver, object.create1);
+      await runCommandOnDriver(conn, driver, object.create2);
       const structure1 = await driver.analyseFull(conn);
-      await driver.query(conn, object.drop2, { discardResult: true });
+      await runCommandOnDriver(conn, driver, object.drop2);
       const structure2 = await driver.analyseIncremental(conn, structure1);
 
       expect(structure2[type].length).toEqual(1);
@@ -78,11 +89,11 @@ describe('Object analyse', () => {
   test.each(flatSource())(
     'Create SQL - add - %s - %s',
     testWrapper(async (conn, driver, type, object, engine) => {
-      for (const sql of initSql) await driver.query(conn, sql, { discardResult: true });
+      for (const sql of initSql) await runCommandOnDriver(conn, driver, sql);
 
-      await driver.query(conn, object.create1, { discardResult: true });
+      await runCommandOnDriver(conn, driver, object.create1);
       const structure1 = await driver.analyseFull(conn);
-      await driver.query(conn, object.drop1, { discardResult: true });
+      await runCommandOnDriver(conn, driver, object.drop1);
       const structure2 = await driver.analyseIncremental(conn, structure1);
       expect(structure2[type].length).toEqual(0);
 
@@ -95,44 +106,94 @@ describe('Object analyse', () => {
     })
   );
 
-  test.each(flatSourceParameters())(
-    'Test parameters simple analyse - %s - %s',
-    testWrapper(async (conn, driver, testName, parameter, engine) => {
-      for (const sql of initSql) await driver.query(conn, sql, { discardResult: true });
-      for (const sql of engine.parametersOtherSql) await driver.query(conn, sql, { discardResult: true });
+  const flatParameters = flatSourceParameters();
 
-      await driver.query(conn, parameter.create, { discardResult: true });
+  if (flatParameters.length > 0) {
+    test.each(flatParameters)(
+      'Test parameters simple analyse - %s - %s',
+      testWrapper(async (conn, driver, testName, parameter, engine) => {
+        for (const sql of initSql) await runCommandOnDriver(conn, driver, sql);
+        for (const sql of engine.parametersOtherSql) await runCommandOnDriver(conn, driver, sql);
+
+        await runCommandOnDriver(conn, driver, parameter.create);
+        const structure = await driver.analyseFull(conn);
+
+        const parameters = structure[parameter.objectTypeField].find(x => x.pureName == 'obj1').parameters;
+
+        expect(parameters.length).toEqual(parameter.list.length);
+        for (let i = 0; i < parameters.length; i += 1) {
+          expect(parameters[i]).toEqual(expect.objectContaining(parameter.list[i]));
+        }
+      })
+    );
+
+    test.each(flatParameters)(
+      'Test parameters create SQL - %s - %s',
+      testWrapper(async (conn, driver, testName, parameter, engine) => {
+        for (const sql of initSql) await runCommandOnDriver(conn, driver, sql);
+        for (const sql of engine.parametersOtherSql) await runCommandOnDriver(conn, driver, sql);
+
+        await runCommandOnDriver(conn, driver, parameter.create);
+        const structure1 = await driver.analyseFull(conn);
+        await runCommandOnDriver(conn, driver, parameter.drop);
+
+        const obj = structure1[parameter.objectTypeField].find(x => x.pureName == 'obj1');
+        await driver.script(conn, obj.createSql, { discardResult: true });
+
+        const structure2 = await driver.analyseFull(conn);
+        const parameters = structure2[parameter.objectTypeField].find(x => x.pureName == 'obj1').parameters;
+
+        expect(parameters.length).toEqual(parameter.list.length);
+        for (let i = 0; i < parameters.length; i += 1) {
+          expect(parameters[i]).toEqual(expect.objectContaining(parameter.list[i]));
+        }
+      })
+    );
+  }
+
+  test.each(flatSourceTriggers())(
+    'Test triggers - %s - %s',
+    testWrapper(async (conn, driver, trigger) => {
+      for (const sql of initSql) await runCommandOnDriver(conn, driver, sql);
+
+      const { triggerOtherDropSql, triggerOtherCreateSql, create, drop, expected, objectTypeField } = trigger;
+
+      if (triggerOtherCreateSql) await runCommandOnDriver(conn, driver, triggerOtherCreateSql);
+
+      await runCommandOnDriver(conn, driver, create);
       const structure = await driver.analyseFull(conn);
+      await runCommandOnDriver(conn, driver, drop);
 
-      const parameters = structure[parameter.objectTypeField].find(x => x.pureName == 'obj1').parameters;
+      if (triggerOtherDropSql) await runCommandOnDriver(conn, driver, triggerOtherDropSql);
 
-      expect(parameters.length).toEqual(parameter.list.length);
-      for (let i = 0; i < parameters.length; i += 1) {
-        expect(parameters[i]).toEqual(expect.objectContaining(parameter.list[i]));
-      }
-    })
-  );
+      const createdTrigger = structure[objectTypeField].find(x => x.pureName == expected.pureName);
+      expect(createdTrigger).toEqual(expect.objectContaining(expected));
 
-  test.each(flatSourceParameters())(
-    'Test parameters create SQL - %s - %s',
-    testWrapper(async (conn, driver, testName, parameter, engine) => {
-      for (const sql of initSql) await driver.query(conn, sql, { discardResult: true });
-      for (const sql of engine.parametersOtherSql) await driver.query(conn, sql, { discardResult: true });
-
-      await driver.query(conn, parameter.create, { discardResult: true });
-      const structure1 = await driver.analyseFull(conn);
-      await driver.query(conn, parameter.drop, { discardResult: true });
-
-      const obj = structure1[parameter.objectTypeField].find(x => x.pureName == 'obj1');
-      await driver.script(conn, obj.createSql);
+      // test trigger createSql
+      if (triggerOtherCreateSql) await runCommandOnDriver(conn, driver, triggerOtherCreateSql);
 
+      await driver.script(conn, createdTrigger.createSql);
       const structure2 = await driver.analyseFull(conn);
-      const parameters = structure2[parameter.objectTypeField].find(x => x.pureName == 'obj1').parameters;
-
-      expect(parameters.length).toEqual(parameter.list.length);
-      for (let i = 0; i < parameters.length; i += 1) {
-        expect(parameters[i]).toEqual(expect.objectContaining(parameter.list[i]));
-      }
+      const createdTrigger2 = structure2[objectTypeField].find(x => x.pureName == expected.pureName);
+      expect(createdTrigger2).toEqual(expect.objectContaining(expected));
     })
   );
+
+  const schedulerEvents = flatSourceSchedulerEvents();
+  if (schedulerEvents.length > 0) {
+    test.each(schedulerEvents)(
+      'Test scheduler events - %s - %s',
+      testWrapper(async (conn, driver, event) => {
+        for (const sql of initSql) await runCommandOnDriver(conn, driver, sql);
+        const { create, drop, objectTypeField, expected } = event;
+
+        await runCommandOnDriver(conn, driver, create);
+        const structure = await driver.analyseFull(conn);
+        await runCommandOnDriver(conn, driver, drop);
+
+        const createdEvent = structure[objectTypeField].find(x => x.pureName == expected.pureName);
+        expect(createdEvent).toEqual(expect.objectContaining(expected));
+      })
+    );
+  }
 });

integration-tests/__tests__/query.spec.js

@@ -1,11 +1,12 @@
 const engines = require('../engines');
 const { splitQuery } = require('dbgate-query-splitter');
 const { testWrapper } = require('../tools');
+const { runQueryOnDriver, runCommandOnDriver, formatQueryWithoutParams } = require('dbgate-tools');
 
 const initSql = [
-  'CREATE TABLE t1 (id int primary key)',
-  'INSERT INTO t1 (id) VALUES (1)',
-  'INSERT INTO t1 (id) VALUES (2)',
+  'CREATE TABLE ~t1 (~id int primary key)',
+  'INSERT INTO ~t1 (~id) VALUES (1)',
+  'INSERT INTO ~t1 (~id) VALUES (2)',
 ];
 
 expect.extend({
@@ -51,7 +52,7 @@ class StreamHandler {
 function executeStreamItem(driver, conn, sql) {
   return new Promise(resolve => {
     const handler = new StreamHandler(resolve);
-    driver.stream(conn, sql, handler);
+    driver.stream(conn, formatQueryWithoutParams(driver, sql), handler);
   });
 }
 
@@ -68,9 +69,11 @@ describe('Query', () => {
   test.each(engines.map(engine => [engine.label, engine]))(
     'Simple query - %s',
     testWrapper(async (conn, driver, engine) => {
-      for (const sql of initSql) await driver.query(conn, sql, { discardResult: true });
+      for (const sql of initSql) {
+        await runCommandOnDriver(conn, driver, dmp => dmp.put(sql));
+      }
 
-      const res = await driver.query(conn, 'SELECT id FROM t1 ORDER BY id');
+      const res = await runQueryOnDriver(conn, driver, dmp => dmp.put('SELECT ~id FROM ~t1 ORDER BY ~id'));
       expect(res.columns).toEqual([
         expect.objectContaining({
           columnName: 'id',
@@ -91,8 +94,11 @@ describe('Query', () => {
   test.each(engines.map(engine => [engine.label, engine]))(
     'Simple stream query - %s',
     testWrapper(async (conn, driver, engine) => {
-      for (const sql of initSql) await driver.query(conn, sql, { discardResult: true });
-      const results = await executeStream(driver, conn, 'SELECT id FROM t1 ORDER BY id');
+      for (const sql of initSql) {
+        await runCommandOnDriver(conn, driver, dmp => dmp.put(sql));
+      }
+
+      const results = await executeStream(driver, conn, 'SELECT ~id FROM ~t1 ORDER BY ~id');
       expect(results.length).toEqual(1);
       const res = results[0];
 
@@ -104,11 +110,14 @@ describe('Query', () => {
   test.each(engines.map(engine => [engine.label, engine]))(
     'More queries - %s',
     testWrapper(async (conn, driver, engine) => {
-      for (const sql of initSql) await driver.query(conn, sql, { discardResult: true });
+      for (const sql of initSql) {
+        await runCommandOnDriver(conn, driver, dmp => dmp.put(sql));
+      }
+
       const results = await executeStream(
         driver,
         conn,
-        'SELECT id FROM t1 ORDER BY id; SELECT id FROM t1 ORDER BY id DESC'
+        'SELECT ~id FROM ~t1 ORDER BY ~id; SELECT ~id FROM ~t1 ORDER BY ~id DESC'
       );
       expect(results.length).toEqual(2);
 
@@ -128,7 +137,7 @@ describe('Query', () => {
       const results = await executeStream(
         driver,
         conn,
-        'CREATE TABLE t1 (id int primary key); INSERT INTO t1 (id) VALUES (1); INSERT INTO t1 (id) VALUES (2); SELECT id FROM t1 ORDER BY id; '
+        'CREATE TABLE ~t1 (~id int primary key); INSERT INTO ~t1 (~id) VALUES (1); INSERT INTO ~t1 (~id) VALUES (2); SELECT ~id FROM ~t1 ORDER BY ~id; '
       );
       expect(results.length).toEqual(1);
 
@@ -144,7 +153,7 @@ describe('Query', () => {
       const results = await executeStream(
         driver,
         conn,
-        'CREATE TABLE t1 (id int); INSERT INTO t1 (id) VALUES (1); INSERT INTO t1 (id) VALUES (2) '
+        'CREATE TABLE ~t1 (~id int); INSERT INTO ~t1 (~id) VALUES (1); INSERT INTO ~t1 (~id) VALUES (2) '
       );
       expect(results.length).toEqual(0);
     })
@@ -153,16 +162,57 @@ describe('Query', () => {
   test.each(engines.filter(x => !x.skipDataModifications).map(engine => [engine.label, engine]))(
     'Save data query - %s',
     testWrapper(async (conn, driver, engine) => {
-      for (const sql of initSql) await driver.query(conn, sql, { discardResult: true });
+      for (const sql of initSql) {
+        await runCommandOnDriver(conn, driver, dmp => dmp.put(sql));
+      }
 
       await driver.script(
         conn,
-        'INSERT INTO t1 (id) VALUES (3);INSERT INTO t1 (id) VALUES (4);UPDATE t1 SET id=10 WHERE id=1;DELETE FROM t1 WHERE id=2;',
+        formatQueryWithoutParams(
+          driver,
+          'INSERT INTO ~t1 (~id) VALUES (3);INSERT INTO ~t1 (~id) VALUES (4);UPDATE ~t1 SET ~id=10 WHERE ~id=1;DELETE FROM ~t1 WHERE ~id=2;'
+        ),
         { discardResult: true }
       );
-      const res = await driver.query(conn, 'SELECT COUNT(*) AS cnt FROM t1');
+      const res = await runQueryOnDriver(conn, driver, dmp => dmp.put('SELECT COUNT(*) AS ~cnt FROM ~t1'));
       // console.log(res);
       expect(res.rows[0].cnt == 3).toBeTruthy();
     })
   );
+
+  test.each(engines.filter(x => !x.skipDataDuplicator).map(engine => [engine.label, engine]))(
+    'Select scope identity - %s',
+    testWrapper(async (conn, driver, engine) => {
+      await runCommandOnDriver(conn, driver, dmp =>
+        dmp.createTable({
+          pureName: 't1',
+          columns: [
+            { columnName: 'id', dataType: 'int', notNull: true, autoIncrement: true },
+            { columnName: 'val', dataType: 'varchar(50)' },
+          ],
+          primaryKey: {
+            columns: [{ columnName: 'id' }],
+          },
+        })
+      );
+
+      const structure = await driver.analyseFull(conn);
+      const table = structure.tables.find(x => x.pureName == 't1');
+
+      let res;
+      if (driver.dialect.requireStandaloneSelectForScopeIdentity) {
+        await runCommandOnDriver(conn, driver, dmp => dmp.put("INSERT INTO ~t1 (~val) VALUES ('aaa')"));
+        res = await runQueryOnDriver(conn, driver, dmp => dmp.selectScopeIdentity(table));
+      } else {
+        res = await runQueryOnDriver(conn, driver, dmp => {
+          dmp.putCmd("INSERT INTO ~t1 (~val) VALUES ('aaa')");
+          dmp.selectScopeIdentity(table);
+        });
+      }
+      const row = res.rows[0];
+      const keys = Object.keys(row);
+      expect(keys.length).toEqual(1);
+      expect(row[keys[0]] == 1).toBeTruthy();
+    })
+  );
 });

integration-tests/__tests__/schema-tests.spec.js

@@ -1,7 +1,7 @@
 const stableStringify = require('json-stable-stringify');
 const _ = require('lodash');
 const fp = require('lodash/fp');
-const { testWrapper, extractConnection } = require('../tools');
+const { testWrapper } = require('../tools');
 const engines = require('../engines');
 const { runCommandOnDriver } = require('dbgate-tools');
 
@@ -62,7 +62,7 @@ describe('Schema tests', () => {
       await runCommandOnDriver(handle, driver, dmp => dmp.createSchema('myschema'));
 
       const schemaConnDef = {
-        ...extractConnection(engine),
+        ...engine.connection,
         database: `${handle.database}::myschema`,
       };
 
@@ -76,7 +76,7 @@ describe('Schema tests', () => {
 });
 
 describe('Base analyser test', () => {
-  test.each(engines.map(engine => [engine.label, engine]))(
+  test.each(engines.filter(x => !x.skipIncrementalAnalysis).map(engine => [engine.label, engine]))(
     'Structure without change - %s',
     testWrapper(async (conn, driver, engine) => {
       await baseStructure(conn, driver);

integration-tests/__tests__/table-analyse.spec.js

@@ -1,12 +1,13 @@
+const { runCommandOnDriver } = require('dbgate-tools');
 const engines = require('../engines');
 const { testWrapper } = require('../tools');
 
-const t1Sql = 'CREATE TABLE t1 (id int not null primary key, val1 varchar(50))';
-const ix1Sql = 'CREATE index ix1 ON t1(val1, id)';
+const t1Sql = 'CREATE TABLE ~t1 (~id int not null primary key, ~val1 varchar(50))';
+const ix1Sql = 'CREATE index ~ix1 ON ~t1(~val1, ~id)';
 const t2Sql = engine =>
-  `CREATE TABLE t2 (id int not null primary key, val2 varchar(50) ${engine.skipUnique ? '' : 'unique'})`;
-const t3Sql = 'CREATE TABLE t3 (id int not null primary key, valfk int, foreign key (valfk) references t2(id))';
-const t4Sql = 'CREATE TABLE t4 (id int not null primary key, valdef int not null default 12)';
+  `CREATE TABLE ~t2 (~id int not null primary key, ~val2 varchar(50) ${engine.skipUnique ? '' : 'unique'})`;
+const t3Sql = 'CREATE TABLE ~t3 (~id int not null primary key, ~valfk int, foreign key (~valfk) references ~t2(~id))';
+const t4Sql = 'CREATE TABLE ~t4 (~id int not null primary key, ~valdef int default 12 not null)';
 // const fkSql = 'ALTER TABLE t3 ADD FOREIGN KEY (valfk) REFERENCES t2(id)'
 
 const txMatch = (engine, tname, vcolname, nextcol, defaultValue) =>
@@ -15,7 +16,7 @@ const txMatch = (engine, tname, vcolname, nextcol, defaultValue) =>
     columns: [
       expect.objectContaining({
         columnName: 'id',
-        dataType: expect.stringMatching(/int.*/i),
+        dataType: expect.stringMatching(/int.*|number/i),
         ...(engine.skipNullability ? {} : { notNull: true }),
       }),
       expect.objectContaining({
@@ -59,9 +60,10 @@ describe('Table analyse', () => {
   test.each(engines.map(engine => [engine.label, engine]))(
     'Table structure - full analysis - %s',
     testWrapper(async (conn, driver, engine) => {
-      await driver.query(conn, t1Sql);
+      await runCommandOnDriver(conn, driver, dmp => dmp.put(t1Sql));
 
       const structure = await driver.analyseFull(conn);
+      console.log(JSON.stringify(structure, null, 2));
 
       expect(structure.tables.length).toEqual(1);
       expect(structure.tables[0]).toEqual(t1Match(engine));
@@ -71,13 +73,13 @@ describe('Table analyse', () => {
   test.each(engines.map(engine => [engine.label, engine]))(
     'Table add - incremental analysis - %s',
     testWrapper(async (conn, driver, engine) => {
-      await driver.query(conn, t2Sql(engine));
+      await runCommandOnDriver(conn, driver, dmp => dmp.put(t2Sql(engine)));
 
       const structure1 = await driver.analyseFull(conn);
       expect(structure1.tables.length).toEqual(1);
       expect(structure1.tables[0]).toEqual(t2Match(engine));
 
-      await driver.query(conn, t1Sql);
+      await runCommandOnDriver(conn, driver, dmp => dmp.put(t1Sql));
       const structure2 = await driver.analyseIncremental(conn, structure1);
 
       expect(structure2.tables.length).toEqual(2);
@@ -89,14 +91,14 @@ describe('Table analyse', () => {
   test.each(engines.map(engine => [engine.label, engine]))(
     'Table remove - incremental analysis - %s',
     testWrapper(async (conn, driver, engine) => {
-      await driver.query(conn, t1Sql);
-      await driver.query(conn, t2Sql(engine));
+      await runCommandOnDriver(conn, driver, dmp => dmp.put(t1Sql));
+      await runCommandOnDriver(conn, driver, dmp => dmp.put(t2Sql(engine)));
       const structure1 = await driver.analyseFull(conn);
       expect(structure1.tables.length).toEqual(2);
       expect(structure1.tables.find(x => x.pureName == 't1')).toEqual(t1Match(engine));
       expect(structure1.tables.find(x => x.pureName == 't2')).toEqual(t2Match(engine));
 
-      await driver.query(conn, 'DROP TABLE t2');
+      await runCommandOnDriver(conn, driver, dmp => dmp.put('DROP TABLE ~t2'));
       const structure2 = await driver.analyseIncremental(conn, structure1);
 
       expect(structure2.tables.length).toEqual(1);
@@ -107,15 +109,14 @@ describe('Table analyse', () => {
   test.each(engines.map(engine => [engine.label, engine]))(
     'Table change - incremental analysis - %s',
     testWrapper(async (conn, driver, engine) => {
-      await driver.query(conn, t1Sql);
-      await driver.query(conn, t2Sql(engine));
+      await runCommandOnDriver(conn, driver, dmp => dmp.put(t1Sql));
+      await runCommandOnDriver(conn, driver, dmp => dmp.put(t2Sql(engine)));
       const structure1 = await driver.analyseFull(conn);
 
       if (engine.dbSnapshotBySeconds) await new Promise(resolve => setTimeout(resolve, 1100));
 
-      await driver.query(
-        conn,
-        `ALTER TABLE t2 ADD ${engine.alterTableAddColumnSyntax ? 'COLUMN' : ''} nextcol varchar(50)`
+      await runCommandOnDriver(conn, driver, dmp =>
+        dmp.put(`ALTER TABLE ~t2 ADD ${engine.alterTableAddColumnSyntax ? 'COLUMN' : ''} ~nextcol varchar(50)`)
       );
       const structure2 = await driver.analyseIncremental(conn, structure1);
 
@@ -130,8 +131,8 @@ describe('Table analyse', () => {
   test.each(engines.filter(x => !x.skipIndexes).map(engine => [engine.label, engine]))(
     'Index - full analysis - %s',
     testWrapper(async (conn, driver, engine) => {
-      await driver.query(conn, t1Sql);
-      await driver.query(conn, ix1Sql);
+      await runCommandOnDriver(conn, driver, dmp => dmp.put(t1Sql));
+      await runCommandOnDriver(conn, driver, dmp => dmp.put(ix1Sql));
       const structure = await driver.analyseFull(conn);
 
       const t1 = structure.tables.find(x => x.pureName == 't1');
@@ -145,7 +146,7 @@ describe('Table analyse', () => {
   test.each(engines.filter(x => !x.skipUnique).map(engine => [engine.label, engine]))(
     'Unique - full analysis - %s',
     testWrapper(async (conn, driver, engine) => {
-      await driver.query(conn, t2Sql(engine));
+      await runCommandOnDriver(conn, driver, dmp => dmp.put(t2Sql(engine)));
       const structure = await driver.analyseFull(conn);
 
       const t2 = structure.tables.find(x => x.pureName == 't2');
@@ -159,8 +160,8 @@ describe('Table analyse', () => {
   test.each(engines.filter(x => !x.skipReferences).map(engine => [engine.label, engine]))(
     'Foreign key - full analysis - %s',
     testWrapper(async (conn, driver, engine) => {
-      await driver.query(conn, t2Sql(engine));
-      await driver.query(conn, t3Sql);
+      await runCommandOnDriver(conn, driver, dmp => dmp.put(t2Sql(engine)));
+      await runCommandOnDriver(conn, driver, dmp => dmp.put(t3Sql));
       // await driver.query(conn, fkSql);
 
       const structure = await driver.analyseFull(conn);
@@ -179,7 +180,7 @@ describe('Table analyse', () => {
   test.each(engines.map(engine => [engine.label, engine]))(
     'Table structure - default value - %s',
     testWrapper(async (conn, driver, engine) => {
-      await driver.query(conn, t4Sql);
+      await runCommandOnDriver(conn, driver, dmp => dmp.put(t4Sql));
 
       const structure = await driver.analyseFull(conn);
 

integration-tests/__tests__/table-create.spec.js

@@ -2,7 +2,7 @@ const _ = require('lodash');
 const fp = require('lodash/fp');
 const engines = require('../engines');
 const { testWrapper } = require('../tools');
-const { extendDatabaseInfo } = require('dbgate-tools');
+const { extendDatabaseInfo, runCommandOnDriver } = require('dbgate-tools');
 
 function createExpector(value) {
   return _.cloneDeepWith(value, x => {
@@ -25,7 +25,7 @@ function checkTableStructure2(t1, t2) {
 }
 
 async function testTableCreate(conn, driver, table) {
-  await driver.query(conn, `create table t0 (id int not null primary key)`);
+  await runCommandOnDriver(conn, driver, dmp => dmp.put('create table ~t0 (~id int not null primary key)'));
 
   const dmp = driver.createDumper();
   const table1 = {

integration-tests/docker-compose.yaml

@@ -70,3 +70,9 @@ services:
 #       - cockroachdb
 #     restart: on-failure
 
+  oracle:
+    image: gvenzl/oracle-xe:21-slim
+    environment:
+      ORACLE_PASSWORD: Pwd2020Db
+    ports:
+      - 15006:1521

integration-tests/package.json

@@ -1,6 +1,6 @@
 {
   "name": "dbgate-integration-tests",
-  "version": "5.0.0-alpha.1",
+  "version": "6.0.0-alpha.1",
   "homepage": "https://dbgate.org/",
   "repository": {
     "type": "git",
@@ -11,14 +11,11 @@
   "scripts": {
     "wait:local": "cross-env DEVMODE=1 LOCALTEST=1 node wait.js",
     "wait:ci": "cross-env DEVMODE=1 CITEST=1 node wait.js",
-    "test:local": "cross-env DEVMODE=1 LOCALTEST=1 jest",
+    "test:local": "cross-env DEVMODE=1 LOCALTEST=1 jest  --testTimeout=5000",
     "test:local:path": "cross-env DEVMODE=1 LOCALTEST=1 jest --runTestsByPath __tests__/data-duplicator.spec.js",
-    "test:ci": "cross-env DEVMODE=1 CITEST=1 jest --runInBand --json --outputFile=result.json --testLocationInResults --detectOpenHandles --forceExit",
+    "test:ci": "cross-env DEVMODE=1 CITEST=1 jest --runInBand --json --outputFile=result.json --testLocationInResults --detectOpenHandles --forceExit --testTimeout=10000",
     "run:local": "docker-compose down && docker-compose up -d && yarn wait:local && yarn test:local"
   },
-  "jest": {
-    "testTimeout": 5000
-  },
   "devDependencies": {
     "cross-env": "^7.0.3",
     "jest": "^27.0.1",

integration-tests/tools.js

@@ -1,27 +1,16 @@
 const requireEngineDriver = require('dbgate-api/src/utility/requireEngineDriver');
 const crypto = require('crypto');
 
-function randomDbName() {
+function randomDbName(dialect) {
   const generatedKey = crypto.randomBytes(6);
   const newKey = generatedKey.toString('hex');
-  return `db${newKey}`;
-}
-
-function extractConnection(engine) {
-  const { connection } = engine;
-
-  if (process.env.LOCALTEST && engine.local) {
-    return {
-      ...connection,
-      ...engine.local,
-    };
-  }
-
-  return connection;
+  const res = `db${newKey}`;
+  if (dialect.upperCaseAllDbObjectNames) return res.toUpperCase();
+  return res;
 }
 
 async function connect(engine, database) {
-  const connection = extractConnection(engine);
+  const { connection } = engine;
   const driver = requireEngineDriver(connection);
 
   if (engine.generateDbFile) {
@@ -32,19 +21,21 @@ async function connect(engine, database) {
     return conn;
   } else {
     const conn = await driver.connect(connection);
-    await driver.query(conn, `CREATE DATABASE ${database}`);
+    const dmp = driver.createDumper();
+    dmp.createDatabase(database);
+    await driver.query(conn, dmp.s);
     await driver.close(conn);
 
     const res = await driver.connect({
       ...connection,
-      database,
+      database: (driver.dialect.userDatabaseNamePrefix ?? '') + database,
     });
     return res;
   }
 }
 
 async function prepareConnection(engine, database) {
-  const connection = extractConnection(engine);
+  const { connection } = engine;
   const driver = requireEngineDriver(connection);
 
   if (engine.generateDbFile) {
@@ -55,12 +46,14 @@ async function prepareConnection(engine, database) {
     };
   } else {
     const conn = await driver.connect(connection);
-    await driver.query(conn, `CREATE DATABASE ${database}`);
+    const dmp = driver.createDumper();
+    dmp.createDatabase(database);
+    await driver.query(conn, dmp.s);
     await driver.close(conn);
 
     return {
       ...connection,
-      database,
+      database: (driver.dialect.userDatabaseNamePrefix ?? '') + database,
       isPreparedOnly: true,
     };
   }
@@ -71,7 +64,7 @@ const testWrapper =
   async (label, ...other) => {
     const engine = other[other.length - 1];
     const driver = requireEngineDriver(engine.connection);
-    const conn = await connect(engine, randomDbName());
+    const conn = await connect(engine, randomDbName(driver.dialect));
     try {
       await body(conn, driver, ...other);
     } finally {
@@ -84,14 +77,13 @@ const testWrapperPrepareOnly =
   async (label, ...other) => {
     const engine = other[other.length - 1];
     const driver = requireEngineDriver(engine.connection);
-    const conn = await prepareConnection(engine, randomDbName());
+    const conn = await prepareConnection(engine, randomDbName(driver.dialect));
     await body(conn, driver, ...other);
   };
 
 module.exports = {
   randomDbName,
   connect,
-  extractConnection,
   testWrapper,
   testWrapperPrepareOnly,
 };

integration-tests/wait.js

@@ -1,13 +1,12 @@
 const requireEngineDriver = require('dbgate-api/src/utility/requireEngineDriver');
 const engines = require('./engines');
-const { extractConnection } = require('./tools');
 global.DBGATE_PACKAGES = {
   'dbgate-tools': require('dbgate-tools'),
   'dbgate-sqltree': require('dbgate-sqltree'),
 };
 
 async function connectEngine(engine) {
-  const connection = extractConnection(engine);
+  const { connection } = engine;
   const driver = requireEngineDriver(connection);
   for (;;) {
     try {

package.json

@@ -1,6 +1,6 @@
 {
   "private": true,
-  "version": "5.5.7-beta.66",
+  "version": "6.1.3-beta.2",
   "name": "dbgate-all",
   "workspaces": [
     "packages/*",
@@ -47,13 +47,13 @@
     "setCurrentVersion": "node setCurrentVersion",
     "printSecrets": "node printSecrets",
     "generatePadFile": "node generatePadFile",
-    "adjustPackageJson": "node adjustPackageJson",
     "fillPackagedPlugins": "node fillPackagedPlugins",
     "resetPackagedPlugins": "node resetPackagedPlugins",
     "prettier": "prettier --write packages/api/src && prettier --write packages/datalib/src && prettier --write packages/filterparser/src && prettier --write packages/sqltree/src && prettier --write packages/tools/src && prettier --write packages/types && prettier --write packages/web/src && prettier --write app/src",
     "copy:docker:build": "copyfiles packages/api/dist/* docker -f && copyfiles packages/web/public/* docker -u 2 && copyfiles \"packages/web/public/**/*\" docker -u 2 && copyfiles \"plugins/dist/**/*\" docker/plugins -u 2",
-    "copy:packer:build": "copyfiles packages/api/dist/* packer/build -f && copyfiles packages/web/public/* packer/build -u 2 && copyfiles \"packages/web/public/**/*\" packer/build -u 2 && copyfiles \"plugins/dist/**/*\" packer/build/plugins -u 2 && copyfiles packer/install-packages.sh packer/build -f",
+    "copy:packer:build": "copyfiles packages/api/dist/* packer/build -f && copyfiles packages/web/public/* packer/build -u 2 && copyfiles \"packages/web/public/**/*\" packer/build -u 2 && copyfiles \"plugins/dist/**/*\" packer/build/plugins -u 2 && copyfiles packer/install-packages.sh packer/build -f && yarn install:drivers:packer",
     "install:drivers:docker": "node common/defineVolatileDependencies.js docker && cd docker && yarn install && cd ..",
+    "install:drivers:packer": "node common/defineVolatileDependencies.js packer/build",
     "prepare:docker": "yarn plugins:copydist && yarn build:web && yarn build:api && yarn copy:docker:build && yarn install:drivers:docker",
     "prepare:packer": "yarn plugins:copydist && yarn build:web && yarn build:api && yarn copy:packer:build",
     "start": "concurrently --kill-others-on-fail \"yarn start:api\" \"yarn start:web\"",
@@ -62,7 +62,8 @@
     "ts:web": "yarn workspace dbgate-web ts",
     "ts": "yarn ts:api && yarn ts:web",
     "postinstall": "yarn resetPackagedPlugins && yarn build:lib && patch-package && yarn build:plugins:frontend",
-    "dbgate-serve": "node packages/dbgate/bin/dbgate-serve.js"
+    "dbgate-serve": "node packages/dbgate/bin/dbgate-serve.js",
+    "workflows": "node common/processWorkflows.js"
   },
   "dependencies": {
     "concurrently": "^5.1.0",

packages/api/package.json

@@ -1,7 +1,7 @@
 {
   "name": "dbgate-api",
   "main": "src/index.js",
-  "version": "5.0.0-alpha.1",
+  "version": "6.0.0-alpha.1",
   "homepage": "https://dbgate.org/",
   "repository": {
     "type": "git",
@@ -29,10 +29,10 @@
     "compare-versions": "^3.6.0",
     "cors": "^2.8.5",
     "cross-env": "^6.0.3",
-    "dbgate-datalib": "^5.0.0-alpha.1",
-    "dbgate-query-splitter": "^4.11.2",
-    "dbgate-sqltree": "^5.0.0-alpha.1",
-    "dbgate-tools": "^5.0.0-alpha.1",
+    "dbgate-datalib": "^6.0.0-alpha.1",
+    "dbgate-query-splitter": "^4.11.3",
+    "dbgate-sqltree": "^6.0.0-alpha.1",
+    "dbgate-tools": "^6.0.0-alpha.1",
     "debug": "^4.3.4",
     "diff": "^5.0.0",
     "diff2html": "^3.4.13",
@@ -60,7 +60,7 @@
     "rimraf": "^3.0.0",
     "semver": "^7.6.3",
     "simple-encryptor": "^4.0.0",
-    "ssh2": "^1.11.0",
+    "ssh2": "^1.16.0",
     "stream-json": "^1.8.0",
     "tar": "^6.0.5"
   },
@@ -81,7 +81,7 @@
   "devDependencies": {
     "@types/fs-extra": "^9.0.11",
     "@types/lodash": "^4.14.149",
-    "dbgate-types": "^5.0.0-alpha.1",
+    "dbgate-types": "^6.0.0-alpha.1",
     "env-cmd": "^10.1.0",
     "jsdoc-to-markdown": "^9.0.5",
     "node-loader": "^1.0.2",

packages/api/src/auth/authProvider.js

@@ -230,7 +230,7 @@ class LoginsProvider extends AuthProviderBase {
         ),
       };
     }
-    
+
     return { error: 'Invalid credentials' };
   }
 
@@ -271,11 +271,10 @@ function hasEnvLogins() {
   return false;
 }
 
-function detectEnvAuthProvider() {
+function detectEnvAuthProviderCore() {
   if (process.env.AUTH_PROVIDER) {
     return process.env.AUTH_PROVIDER;
   }
-
   if (process.env.STORAGE_DATABASE) {
     return 'denyall';
   }
@@ -291,6 +290,14 @@ function detectEnvAuthProvider() {
   return 'none';
 }
 
+function detectEnvAuthProvider() {
+  const authProvider = detectEnvAuthProviderCore();
+  if (process.env.BASIC_AUTH && authProvider != 'logins' && authProvider != 'ad') {
+    throw new Error(`BASIC_AUTH is not supported with ${authProvider} auth provider`);
+  }
+  return authProvider;
+}
+
 function createEnvAuthProvider() {
   const authProvider = detectEnvAuthProvider();
   switch (authProvider) {

packages/api/src/controllers/config.js

@@ -9,6 +9,7 @@ const _ = require('lodash');
 const AsyncLock = require('async-lock');
 const jwt = require('jsonwebtoken');
 
+const processArgs = require('../utility/processArgs');
 const currentVersion = require('../currentVersion');
 const platformInfo = require('../utility/platformInfo');
 const connections = require('../controllers/connections');
@@ -62,6 +63,8 @@ module.exports = {
     const logoutUrl = storageConnectionError ? null : await authProvider.getLogoutUrl();
     const adminConfig = storageConnectionError ? null : await storage.readConfig({ group: 'admin' });
 
+    storage.startRefreshLicense();
+
     const isAdminPasswordMissing = !!(
       process.env.STORAGE_DATABASE &&
       !process.env.ADMIN_PASSWORD &&
@@ -81,7 +84,8 @@ module.exports = {
       isElectron: platformInfo.isElectron,
       isLicenseValid,
       isLicenseExpired: checkedLicense?.isExpired,
-      trialDaysLeft: checkedLicense?.isGeneratedTrial && !checkedLicense?.isExpired ? checkedLicense?.daysLeft : null,
+      trialDaysLeft:
+        checkedLicense?.licenseTypeObj?.isTrial && !checkedLicense?.isExpired ? checkedLicense?.daysLeft : null,
       checkedLicense,
       configurationError,
       logoutUrl,
@@ -99,7 +103,10 @@ module.exports = {
       adminPasswordState: adminConfig?.adminPasswordState,
       storageDatabase: process.env.STORAGE_DATABASE,
       logsFilePath: getLogsFilePath(),
-      connectionsFilePath: path.join(datadir(), 'connections.jsonl'),
+      connectionsFilePath: path.join(
+        datadir(),
+        processArgs.runE2eTests ? 'connections-e2etests.jsonl' : 'connections.jsonl'
+      ),
       ...currentVersion,
     };
 

packages/api/src/controllers/connections.js

@@ -199,8 +199,11 @@ module.exports = {
     const dir = datadir();
     if (!portalConnections) {
       // @ts-ignore
-      this.datastore = new JsonLinesDatabase(path.join(dir, 'connections.jsonl'));
+      this.datastore = new JsonLinesDatabase(
+        path.join(dir, processArgs.runE2eTests ? 'connections-e2etests.jsonl' : 'connections.jsonl')
+      );
     }
+    await this.checkUnsavedConnectionsLimit();
   },
 
   list_meta: true,
@@ -219,7 +222,7 @@ module.exports = {
   },
 
   test_meta: true,
-  test(connection) {
+  test({ connection, requestDbList = false }) {
     const subprocess = fork(
       global['API_PACKAGE'] || process.argv[1],
       [
@@ -234,7 +237,7 @@ module.exports = {
       }
     );
     pipeForkLogs(subprocess);
-    subprocess.send(connection);
+    subprocess.send({ ...connection, requestDbList });
     return new Promise(resolve => {
       subprocess.on('message', resp => {
         if (handleProcessCommunication(resp, subprocess)) return;
@@ -264,7 +267,7 @@ module.exports = {
     }
 
     if (test) {
-      const testRes = await this.test(res);
+      const testRes = await this.test({ connection: res });
       if (testRes.msgtype == 'connected') {
         volatileConnections[res._id] = res;
         return {
@@ -300,6 +303,32 @@ module.exports = {
     return res;
   },
 
+  async checkUnsavedConnectionsLimit() {
+    if (!this.datastore) {
+      return;
+    }
+    const MAX_UNSAVED_CONNECTIONS = 5;
+    await this.datastore.transformAll(connections => {
+      const count = connections.filter(x => x.unsaved).length;
+      if (count > MAX_UNSAVED_CONNECTIONS) {
+        const res = [];
+        let unsavedToSkip = count - MAX_UNSAVED_CONNECTIONS;
+        for (const item of connections) {
+          if (item.unsaved) {
+            if (unsavedToSkip > 0) {
+              unsavedToSkip--;
+            } else {
+              res.push(item);
+            }
+          } else {
+            res.push(item);
+          }
+        }
+        return res;
+      }
+    });
+  },
+
   update_meta: true,
   async update({ _id, values }, req) {
     if (portalConnections) return;
@@ -483,4 +512,10 @@ module.exports = {
     }
     return null;
   },
+
+  reloadConnectionList_meta: true,
+  async reloadConnectionList() {
+    if (portalConnections) return;
+    await this.datastore.unload();
+  },
 };

packages/api/src/controllers/runners.js

@@ -49,7 +49,8 @@ logger.info('Finished job script');
 dbgateApi.runScript(run);
 `;
 
-const loaderScriptTemplate = (functionName, props, runid) => `
+const loaderScriptTemplate = (prefix, functionName, props, runid) => `
+${prefix}
 const dbgateApi = require(process.env.DBGATE_API);
 dbgateApi.initializeApiEnvironment();
 ${requirePluginsTemplate(extractShellApiPlugins(functionName, props))}
@@ -149,7 +150,7 @@ module.exports = {
     byline(subprocess.stdout).on('data', pipeDispatcher('info'));
     byline(subprocess.stderr).on('data', pipeDispatcher('error'));
     subprocess.on('exit', code => {
-      this.rejectRequest(runid, { message: 'No data retured, maybe input data source is too big' });
+      this.rejectRequest(runid, { message: 'No data returned, maybe input data source is too big' });
       logger.info({ code, pid: subprocess.pid }, 'Exited process');
       socket.emit(`runner-done-${runid}`, code);
     });
@@ -224,10 +225,14 @@ module.exports = {
 
   loadReader_meta: true,
   async loadReader({ functionName, props }) {
+    const prefix = extractShellApiPlugins(functionName)
+      .map(packageName => `// @require ${packageName}\n`)
+      .join('');
+
     const promise = new Promise((resolve, reject) => {
       const runid = crypto.randomUUID();
       this.requests[runid] = [resolve, reject];
-      this.startCore(runid, loaderScriptTemplate(functionName, props, runid));
+      this.startCore(runid, loaderScriptTemplate(prefix, functionName, props, runid));
     });
     return promise;
   },

packages/api/src/controllers/storage.js

@@ -26,4 +26,6 @@ module.exports = {
   async readConfig({ group }) {
     return {};
   },
+
+  startRefreshLicense() {},
 };

packages/api/src/currentVersion.js

@@ -1,5 +1,5 @@
 
 module.exports = { 
-  version: '5.0.0-alpha.1',
-  buildTime: '2021-04-17T07:22:49.702Z'
+  version: '6.0.0-alpha.1',
+  buildTime: '2024-12-01T00:00:00Z'
 };

packages/api/src/main.js

@@ -28,6 +28,7 @@ const files = require('./controllers/files');
 const scheduler = require('./controllers/scheduler');
 const queryHistory = require('./controllers/queryHistory');
 const onFinished = require('on-finished');
+const processArgs = require('./utility/processArgs');
 
 const { rundir } = require('./utility/directories');
 const platformInfo = require('./utility/platformInfo');
@@ -77,6 +78,8 @@ function start() {
     app.use(getExpressPath('/'), express.static('/home/dbgate-docker/public'));
   } else if (platformInfo.isAwsUbuntuLayout) {
     app.use(getExpressPath('/'), express.static('/home/ubuntu/build/public'));
+  } else if (processArgs.runE2eTests) {
+    app.use(getExpressPath('/'), express.static(path.resolve('packer/build/public')));
   } else if (platformInfo.isNpmDist) {
     app.use(
       getExpressPath('/'),

packages/api/src/proc/connectProcess.js

@@ -17,12 +17,18 @@ Platform: ${process.platform}
 function start() {
   childProcessChecker();
   process.on('message', async connection => {
+    // @ts-ignore
+    const { requestDbList } = connection;
     if (handleProcessCommunication(connection)) return;
     try {
       const driver = requireEngineDriver(connection);
       const dbhan = await connectUtility(driver, connection, 'app');
       const res = await driver.getVersion(dbhan);
-      process.send({ msgtype: 'connected', ...res });
+      let databases = undefined;
+      if (requestDbList) {
+        databases = await driver.listDatabases(dbhan);
+      }
+      process.send({ msgtype: 'connected', ...res, databases });
       await driver.close(dbhan);
     } catch (e) {
       console.error(e);

packages/api/src/proc/serverConnectionProcess.js

@@ -166,7 +166,9 @@ function start() {
     if (time - lastPing > 40 * 1000) {
       logger.info('Server connection not alive, exiting');
       const driver = requireEngineDriver(storedConnection);
-      await driver.close(dbhan);
+      if (dbhan) {
+        await driver.close(dbhan);
+      }
       process.exit(0);
     }
   }, 10 * 1000);

packages/api/src/proc/sshForwardProcess.js

@@ -7,7 +7,7 @@ const { getLogger, extractErrorLogData, extractErrorMessage } = require('dbgate-
 
 const logger = getLogger('sshProcess');
 
-async function getSshConnection(connection) {
+async function getSshConnection(connection, tunnelConfig) {
   const sshConfig = {
     endHost: connection.sshHost || '',
     endPort: connection.sshPort || 22,
@@ -23,6 +23,7 @@ async function getSshConnection(connection) {
         : undefined,
     skipAutoPrivateKey: true,
     noReadline: true,
+    bindHost: tunnelConfig.fromHost,
   };
 
   const sshConn = new SSHConnection(sshConfig);
@@ -31,7 +32,7 @@ async function getSshConnection(connection) {
 
 async function handleStart({ connection, tunnelConfig }) {
   try {
-    const sshConn = await getSshConnection(connection);
+    const sshConn = await getSshConnection(connection, tunnelConfig);
     await sshConn.forward(tunnelConfig);
 
     process.send({

packages/api/src/shell/tableReader.js

@@ -8,12 +8,15 @@ const logger = getLogger('tableReader');
  * @param {object} options
  * @param {connectionType} options.connection - connection object
  * @param {object} options.systemConnection - system connection (result of driver.connect). If not provided, new connection will be created
+ * @param {object} options.driver - driver object. If not provided, it will be loaded from connection
  * @param {string} options.pureName - table name
  * @param {string} options.schemaName - schema name
  * @returns {Promise<readerType>} - reader object
  */
-async function tableReader({ connection, systemConnection, pureName, schemaName }) {
-  const driver = requireEngineDriver(connection);
+async function tableReader({ connection, systemConnection, pureName, schemaName, driver }) {
+  if (!driver) {
+    driver = requireEngineDriver(connection);
+  }
   const dbhan = systemConnection || (await connectUtility(driver, connection, 'read'));
   logger.info(`Connected.`);
 

packages/api/src/utility/JsonLinesDatabase.js

@@ -23,6 +23,12 @@ class JsonLinesDatabase {
     await fs.writeFile(this.filename, this.data.map(x => JSON.stringify(x)).join('\n'));
   }
 
+  async unload() {
+    this.data = [];
+    this.loadedOk = false;
+    this.loadPerformed = false;
+  }
+
   async _ensureLoaded() {
     if (!this.loadPerformed) {
       await lock.acquire('reader', async () => {
@@ -111,6 +117,15 @@ class JsonLinesDatabase {
     return removed;
   }
 
+  async transformAll(transformFunction) {
+    await this._ensureLoaded();
+    const newData = transformFunction(this.data);
+    if (newData) {
+      this.data = newData;
+      await this._save();
+    }
+  }
+
   //   async _openReader() {
   //     return new Promise((resolve, reject) =>
   //       lineReader.open(this.filename, (err, reader) => {

packages/api/src/utility/SSHConnection.js

@@ -130,7 +130,7 @@ class SSHConnection {
     const connectionToBastion = await this.connect(bastionHost);
     return new Promise((resolve, reject) => {
       connectionToBastion.forwardOut(
-        '127.0.0.1',
+        this.options.bindHost,
         22,
         this.options.endHost,
         this.options.endPort || 22,
@@ -228,9 +228,9 @@ class SSHConnection {
             options.toPort
           );
           connection.forwardOut(
-            'localhost',
+            this.options.bindHost,
             options.fromPort,
-            options.toHost || 'localhost',
+            options.toHost || this.options.bindHost,
             options.toPort,
             (error, stream) => {
               if (error) {
@@ -241,7 +241,7 @@ class SSHConnection {
             }
           );
         })
-        .listen(options.fromPort, 'localhost', () => {
+        .listen(options.fromPort, this.options.bindHost, () => {
           return resolve();
         });
     });

packages/api/src/utility/connectUtility.js

@@ -63,7 +63,7 @@ async function connectUtility(driver, storedConnection, connectionMode, addition
       throw new Error(tunnel.message);
     }
 
-    connection.server = 'localhost';
+    connection.server = tunnel.localHost;
     connection.port = tunnel.localPort;
   }
 

packages/api/src/utility/directories.js

@@ -96,6 +96,9 @@ function packagedPluginsDir() {
     //   return path.resolve(__dirname, '../../plugins');
     // }
   }
+  if (processArgs.runE2eTests) {
+    return path.resolve('packer/build/plugins');
+  }
   return null;
 }
 
@@ -110,7 +113,7 @@ function getPluginBackendPath(packageName) {
     return path.join(packagedPluginsDir(), packageName, 'dist', 'backend.js');
   }
 
-  const res = path.join(pluginsdir(), packageName, 'dist', 'backend.js')
+  const res = path.join(pluginsdir(), packageName, 'dist', 'backend.js');
   if (fs.existsSync(res)) {
     return res;
   }

packages/api/src/utility/processArgs.js

@@ -1,3 +1,5 @@
+const path = require('path');
+
 function getNamedArg(name) {
   const argIndex = process.argv.indexOf(name);
   if (argIndex > 0) {
@@ -14,15 +16,21 @@ const workspaceDir = getNamedArg('--workspace-dir');
 const processDisplayName = getNamedArg('--process-display-name');
 const listenApi = process.argv.includes('--listen-api');
 const listenApiChild = process.argv.includes('--listen-api-child') || listenApi;
+const runE2eTests = process.argv.includes('--run-e2e-tests');
 
 function getPassArgs() {
   const res = [];
   if (global['PLUGINS_DIR']) {
     res.push('--plugins-dir', global['PLUGINS_DIR']);
+  } else if (runE2eTests) {
+    res.push('--plugins-dir', path.resolve('packer/build/plugins'));
   }
   if (listenApiChild) {
     res.push('listen-api-child');
   }
+  // if (runE2eTests) {
+  //   res.push('--run-e2e-tests');
+  // }
   return res;
 }
 
@@ -36,4 +44,5 @@ module.exports = {
   listenApi,
   listenApiChild,
   processDisplayName,
+  runE2eTests,
 };

packages/api/src/utility/sshTunnel.js

@@ -43,33 +43,44 @@ function callForwardProcess(connection, tunnelConfig, tunnelCacheKey) {
     logger.error(extractErrorLogData(err), 'Error connecting SSH');
   }
   return new Promise((resolve, reject) => {
+    let promiseHandled = false;
     subprocess.on('message', resp => {
       // @ts-ignore
       const { msgtype, errorMessage } = resp;
       if (msgtype == 'connected') {
         resolve(subprocess);
+        promiseHandled = true;
       }
       if (msgtype == 'error') {
         reject(new Error(errorMessage));
+        promiseHandled = true;
       }
     });
     subprocess.on('exit', code => {
       logger.info('SSH forward process exited');
       delete sshTunnelCache[tunnelCacheKey];
+      if (!promiseHandled) {
+        reject(new Error('SSH forward process exited, try to change "Local host address for SSH connections" in Settings/Connections'));
+      }
     });
   });
 }
 
 async function getSshTunnel(connection) {
+  const config = require('../controllers/config');
+
   const tunnelCacheKey = stableStringify(_.pick(connection, TUNNEL_FIELDS));
+  const globalSettings = await config.getSettings();
 
   return await lock.acquire(tunnelCacheKey, async () => {
     if (sshTunnelCache[tunnelCacheKey]) return sshTunnelCache[tunnelCacheKey];
     const localPort = await portfinder.getPortPromise({ port: 10000, stopPort: 60000 });
+    const localHost = globalSettings?.['connection.sshBindHost'] || '127.0.0.1';
     // workaround for `getPortPromise` not releasing the port quickly enough
     await new Promise(resolve => setTimeout(resolve, 500));
     const tunnelConfig = {
       fromPort: localPort,
+      fromHost: localHost,
       toPort: connection.port,
       toHost: connection.server,
     };
@@ -87,6 +98,7 @@ async function getSshTunnel(connection) {
       sshTunnelCache[tunnelCacheKey] = {
         state: 'ok',
         localPort,
+        localHost,
         subprocess,
       };
       return sshTunnelCache[tunnelCacheKey];

packages/datalib/package.json

@@ -1,5 +1,5 @@
 {
-  "version": "5.0.0-alpha.1",
+  "version": "6.0.0-alpha.1",
   "name": "dbgate-datalib",
   "main": "lib/index.js",
   "typings": "lib/index.d.ts",
@@ -13,13 +13,13 @@
     "lib"
   ],
   "dependencies": {
-    "dbgate-sqltree": "^5.0.0-alpha.1",
-    "dbgate-tools": "^5.0.0-alpha.1",
-    "dbgate-filterparser": "^5.0.0-alpha.1",
+    "dbgate-sqltree": "^6.0.0-alpha.1",
+    "dbgate-tools": "^6.0.0-alpha.1",
+    "dbgate-filterparser": "^6.0.0-alpha.1",
     "uuid": "^3.4.0"
   },
   "devDependencies": {
-    "dbgate-types": "^5.0.0-alpha.1",
+    "dbgate-types": "^6.0.0-alpha.1",
     "@types/node": "^13.7.0",
     "jest": "^28.1.3",
     "ts-jest": "^28.0.7",

packages/datalib/src/DataDuplicator.ts

@@ -66,7 +66,7 @@ class DuplicatorItemHolder {
     this.autoColumn = this.table.columns.find(x => x.autoIncrement)?.columnName;
     if (
       this.table.primaryKey?.columns?.length != 1 ||
-      this.table.primaryKey?.columns?.[0].columnName != this.autoColumn
+      this.table.primaryKey?.columns?.[0]?.columnName != this.autoColumn
     ) {
       this.autoColumn = null;
     }
@@ -140,6 +140,9 @@ class DuplicatorItemHolder {
           weakref.foreignKey.columns[0].columnName
         );
       });
+      if (this.duplicator.driver.dialect.requireFromDual) {
+        dmp.put(' ^from ^dual');
+      }
     });
     const qrow = qres.rows[0];
     return this.weakReferences.filter(x => qrow[x.columnName] == 0).map(x => x.columnName);
@@ -194,6 +197,7 @@ class DuplicatorItemHolder {
               res = await runQueryOnDriver(pool, driver, dmp => dmp.selectScopeIdentity(this.table));
             }
             // console.log('IDRES', JSON.stringify(res));
+            // console.log('*********** ENTRIES OF', res?.rows?.[0]);
             const resId = Object.entries(res?.rows?.[0])?.[0]?.[1];
             if (resId != null) {
               this.idMap[chunk[this.autoColumn]] = resId;

packages/datalib/src/TableGridDisplay.ts

@@ -37,7 +37,8 @@ export class TableGridDisplay extends GridDisplay {
     public displayOptions: any,
     serverVersion,
     public getDictionaryDescription: DictionaryDescriptionFunc = null,
-    isReadOnly = false
+    isReadOnly = false,
+    public isRawMode = false
   ) {
     super(config, setConfig, cache, setCache, driver, dbinfo, serverVersion);
 
@@ -172,6 +173,9 @@ export class TableGridDisplay extends GridDisplay {
   }
 
   addHintsToSelect(select: Select): boolean {
+    if (this.isRawMode) {
+      return false;
+    }
     let res = false;
     const groupColumns = this.groupColumns;
     for (const column of this.hintBaseColumns || this.getGridColumns()) {

packages/datalib/src/tests/chinookDbInfo.ts

@@ -1492,6 +1492,7 @@ export const chinookDbInfo: DatabaseInfo = {
   collections: [],
   matviews: [],
   triggers: [],
+  schedulerEvents: [],
 };
 
 // const ARTIST_TABLE: TableInfo = {

packages/dbmodel/package.json

@@ -1,6 +1,6 @@
 {
   "name": "dbmodel",
-  "version": "5.0.0-alpha.1",
+  "version": "6.0.0-alpha.1",
   "homepage": "https://dbgate.org/",
   "repository": {
     "type": "git",
@@ -30,16 +30,16 @@
   ],
   "dependencies": {
     "commander": "^10.0.0",
-    "dbgate-api": "^5.0.0-alpha.1",
-    "dbgate-plugin-csv": "^5.0.0-alpha.1",
-    "dbgate-plugin-excel": "^5.0.0-alpha.1",
-    "dbgate-plugin-mongo": "^5.0.0-alpha.1",
-    "dbgate-plugin-mssql": "^5.0.0-alpha.1",
-    "dbgate-plugin-mysql": "^5.0.0-alpha.1",
-    "dbgate-plugin-postgres": "^5.0.0-alpha.1",
-    "dbgate-plugin-xml": "^5.0.0-alpha.1",
-    "dbgate-plugin-oracle": "^5.0.0-alpha.1",
-    "dbgate-web": "^5.0.0-alpha.1",
+    "dbgate-api": "^6.0.0-alpha.1",
+    "dbgate-plugin-csv": "^6.0.0-alpha.1",
+    "dbgate-plugin-excel": "^6.0.0-alpha.1",
+    "dbgate-plugin-mongo": "^6.0.0-alpha.1",
+    "dbgate-plugin-mssql": "^6.0.0-alpha.1",
+    "dbgate-plugin-mysql": "^6.0.0-alpha.1",
+    "dbgate-plugin-postgres": "^6.0.0-alpha.1",
+    "dbgate-plugin-xml": "^6.0.0-alpha.1",
+    "dbgate-plugin-oracle": "^6.0.0-alpha.1",
+    "dbgate-web": "^6.0.0-alpha.1",
     "dotenv": "^16.0.0",
     "pinomin": "^1.0.4"
   }

packages/filterparser/package.json

@@ -1,5 +1,5 @@
 {
-  "version": "5.0.0-alpha.1",
+  "version": "6.0.0-alpha.1",
   "name": "dbgate-filterparser",
   "main": "lib/index.js",
   "typings": "lib/index.d.ts",
@@ -13,7 +13,7 @@
     "lib"
   ],
   "devDependencies": {
-    "dbgate-types": "^5.0.0-alpha.1",
+    "dbgate-types": "^6.0.0-alpha.1",
     "@types/jest": "^25.1.4",
     "@types/node": "^13.7.0",
     "jest": "^28.1.3",
@@ -22,7 +22,7 @@
   },
   "dependencies": {
     "@types/parsimmon": "^1.10.1",
-    "dbgate-tools": "^5.0.0-alpha.1",
+    "dbgate-tools": "^6.0.0-alpha.1",
     "lodash": "^4.17.21",
     "date-fns": "^4.1.0",
     "moment": "^2.24.0",

packages/serve/README.md

@@ -41,7 +41,7 @@ STORAGE_DATABASE=dbname
 STORAGE_ENGINE=mysql@dbgate-plugin-mysql
 ```
 
-You could find more about environment variable configuration on [DbGate docs](https://dbgate.org/docs/env-variables.html) page.
+You could find more about environment variable configuration on [DbGate docs](https://dbgate.org/docs/env-variables/) page.
 
 After installing, you can run dbgate with command:
 ```sh

packages/serve/package.json

@@ -1,6 +1,6 @@
 {
   "name": "dbgate-serve",
-  "version": "5.0.0-alpha.1",
+  "version": "6.0.0-alpha.1",
   "homepage": "https://dbgate.org/",
   "repository": {
     "type": "git",
@@ -18,19 +18,19 @@
     "web"
   ],
   "dependencies": {
-    "dbgate-api": "^5.0.0-alpha.1",
-    "dbgate-plugin-clickhouse": "^5.0.0-alpha.1",
-    "dbgate-plugin-csv": "^5.0.0-alpha.1",
-    "dbgate-plugin-excel": "^5.0.0-alpha.1",
-    "dbgate-plugin-mongo": "^5.0.0-alpha.1",
-    "dbgate-plugin-mssql": "^5.0.0-alpha.1",
-    "dbgate-plugin-mysql": "^5.0.0-alpha.1",
-    "dbgate-plugin-oracle": "^5.0.0-alpha.1",
-    "dbgate-plugin-postgres": "^5.0.0-alpha.1",
-    "dbgate-plugin-redis": "^5.0.0-alpha.1",
-    "dbgate-plugin-sqlite": "^5.0.0-alpha.1",
-    "dbgate-plugin-xml": "^5.0.0-alpha.1",
-    "dbgate-web": "^5.0.0-alpha.1",
+    "dbgate-api": "^6.0.0-alpha.1",
+    "dbgate-plugin-clickhouse": "^6.0.0-alpha.1",
+    "dbgate-plugin-csv": "^6.0.0-alpha.1",
+    "dbgate-plugin-excel": "^6.0.0-alpha.1",
+    "dbgate-plugin-mongo": "^6.0.0-alpha.1",
+    "dbgate-plugin-mssql": "^6.0.0-alpha.1",
+    "dbgate-plugin-mysql": "^6.0.0-alpha.1",
+    "dbgate-plugin-oracle": "^6.0.0-alpha.1",
+    "dbgate-plugin-postgres": "^6.0.0-alpha.1",
+    "dbgate-plugin-redis": "^6.0.0-alpha.1",
+    "dbgate-plugin-sqlite": "^6.0.0-alpha.1",
+    "dbgate-plugin-xml": "^6.0.0-alpha.1",
+    "dbgate-web": "^6.0.0-alpha.1",
     "dotenv": "^16.0.0"
   }
 }

packages/sqltree/package.json

@@ -1,5 +1,5 @@
 {
-  "version": "5.0.0-alpha.1",
+  "version": "6.0.0-alpha.1",
   "name": "dbgate-sqltree",
   "main": "lib/index.js",
   "typings": "lib/index.d.ts",
@@ -27,7 +27,7 @@
   ],
   "devDependencies": {
     "@types/node": "^13.7.0",
-    "dbgate-types": "^5.0.0-alpha.1",
+    "dbgate-types": "^6.0.0-alpha.1",
     "typescript": "^4.4.3"
   },
   "dependencies": {

packages/tools/package.json

@@ -1,5 +1,5 @@
 {
-  "version": "5.0.0-alpha.1",
+  "version": "6.0.0-alpha.1",
   "name": "dbgate-tools",
   "main": "lib/index.js",
   "typings": "lib/index.d.ts",
@@ -25,14 +25,14 @@
   ],
   "devDependencies": {
     "@types/node": "^13.7.0",
-    "dbgate-types": "^5.0.0-alpha.1",
+    "dbgate-types": "^6.0.0-alpha.1",
     "jest": "^24.9.0",
     "ts-jest": "^25.2.1",
     "typescript": "^4.4.3"
   },
   "dependencies": {
-    "dbgate-query-splitter": "^4.11.2",
-    "dbgate-sqltree": "^5.0.0-alpha.1",
+    "dbgate-query-splitter": "^4.11.3",
+    "dbgate-sqltree": "^6.0.0-alpha.1",
     "debug": "^4.3.4",
     "json-stable-stringify": "^1.0.1",
     "lodash": "^4.17.21",

packages/tools/src/DatabaseAnalyser.ts

@@ -10,7 +10,16 @@ import { extractErrorLogData } from './stringTools';
 
 const logger = getLogger('dbAnalyser');
 
-const STRUCTURE_FIELDS = ['tables', 'collections', 'views', 'matviews', 'functions', 'procedures', 'triggers'];
+const STRUCTURE_FIELDS = [
+  'tables',
+  'collections',
+  'views',
+  'matviews',
+  'functions',
+  'procedures',
+  'triggers',
+  'schedulerEvents',
+];
 
 const fp_pick = arg => array => _pick(array, arg);
 
@@ -70,7 +79,9 @@ export class DatabaseAnalyser {
   }
 
   async fullAnalysis() {
-    logger.debug(`Performing full analysis, DB=${dbNameLogCategory(this.dbhan.database)}, engine=${this.driver.engine}`);
+    logger.debug(
+      `Performing full analysis, DB=${dbNameLogCategory(this.dbhan.database)}, engine=${this.driver.engine}`
+    );
     const res = this.addEngineField(await this._runAnalysis());
     // console.log('FULL ANALYSIS', res);
     return res;
@@ -255,6 +266,7 @@ export class DatabaseAnalyser {
       ...this.getDeletedObjectsForField(snapshot, 'procedures'),
       ...this.getDeletedObjectsForField(snapshot, 'functions'),
       ...this.getDeletedObjectsForField(snapshot, 'triggers'),
+      ...this.getDeletedObjectsForField(snapshot, 'schedulerEvents'),
     ];
   }
 
@@ -355,11 +367,12 @@ export class DatabaseAnalyser {
       functions: [],
       procedures: [],
       triggers: [],
+      schedulerEvents: [],
     };
   }
 
   static byTableFilter(table) {
-    return x => x.pureName == table.pureName && x.schemaName == x.schemaName;
+    return x => x.pureName == table.pureName && x.schemaName == table.schemaName;
   }
 
   static extractPrimaryKeys(table, pkColumns) {

packages/tools/src/SqlDumper.ts

@@ -17,6 +17,8 @@ import type {
   CheckInfo,
   AlterProcessor,
   SqlObjectInfo,
+  CallableObjectInfo,
+  SchedulerEventInfo,
 } from 'dbgate-types';
 import _isString from 'lodash/isString';
 import _isNumber from 'lodash/isNumber';
@@ -430,6 +432,14 @@ export class SqlDumper implements AlterProcessor {
   changeTriggerSchema(obj: TriggerInfo, newSchema: string) {}
   renameTrigger(obj: TriggerInfo, newSchema: string) {}
 
+  createSchedulerEvent(obj: SchedulerEventInfo) {
+    this.putRaw(obj.createSql);
+    this.endCommand();
+  }
+  dropSchedulerEvent(obj: SchedulerEventInfo, { testIfExists = false }) {
+    this.putCmd('^drop ^event  %f', obj);
+  }
+
   dropConstraintCore(cnt: ConstraintInfo) {
     this.putCmd('^alter ^table %f ^drop ^constraint %i', cnt, cnt.constraintName);
   }
@@ -707,6 +717,8 @@ export class SqlDumper implements AlterProcessor {
         return 'TRIGGER';
       case 'matviews':
         return 'MATERIALIZED VIEW';
+      case 'schedulerEvents':
+        return 'EVENT';
     }
   }
 
@@ -782,4 +794,15 @@ export class SqlDumper implements AlterProcessor {
       this.endCommand();
     }
   }
+
+  callableTemplate(func: CallableObjectInfo) {
+    this.put('^call %f(&>&n', func);
+
+    this.putCollection(',&n', func.parameters || [], param => {
+      this.putRaw(param.parameterMode == 'IN' ? ':' + param.parameterName : param.parameterName);
+    });
+
+    this.put('&<&n)');
+    this.endCommand();
+  }
 }

packages/tools/src/SqlGenerator.ts

@@ -3,6 +3,7 @@ import type {
   EngineDriver,
   FunctionInfo,
   ProcedureInfo,
+  SchedulerEventInfo,
   TableInfo,
   TriggerInfo,
   ViewInfo,
@@ -49,12 +50,16 @@ interface SqlGeneratorOptions {
   dropTriggers: boolean;
   checkIfTriggerExists: boolean;
   createTriggers: boolean;
+
+  dropSchedulerEvents: boolean;
+  checkIfSchedulerEventExists: boolean;
+  createSchedulerEvents: boolean;
 }
 
 interface SqlGeneratorObject {
   schemaName: string;
   pureName: string;
-  objectTypeField: 'tables' | 'views' | 'procedures' | 'functions';
+  objectTypeField: 'tables' | 'views' | 'procedures' | 'functions' | 'triggers' | 'schedulerEvents';
 }
 
 export class SqlGenerator {
@@ -64,6 +69,7 @@ export class SqlGenerator {
   private procedures: ProcedureInfo[];
   private functions: FunctionInfo[];
   private triggers: TriggerInfo[];
+  private schedulerEvents: SchedulerEventInfo[];
   public dbinfo: DatabaseInfo;
   public isTruncated = false;
   public isUnhandledException = false;
@@ -83,6 +89,7 @@ export class SqlGenerator {
     this.procedures = this.extract('procedures');
     this.functions = this.extract('functions');
     this.triggers = this.extract('triggers');
+    this.schedulerEvents = this.extract('schedulerEvents');
   }
 
   private handleException = error => {
@@ -104,6 +111,8 @@ export class SqlGenerator {
       if (this.checkDumper()) return;
       this.dropObjects(this.triggers, 'Trigger');
       if (this.checkDumper()) return;
+      this.dropObjects(this.schedulerEvents, 'SchedulerEvent');
+      if (this.checkDumper()) return;
 
       this.dropTables();
       if (this.checkDumper()) return;
@@ -130,6 +139,8 @@ export class SqlGenerator {
       if (this.checkDumper()) return;
       this.createObjects(this.triggers, 'Trigger');
       if (this.checkDumper()) return;
+      this.createObjects(this.schedulerEvents, 'SchedulerEvent');
+      if (this.checkDumper()) return;
     } finally {
       process.off('uncaughtException', this.handleException);
     }

packages/tools/src/diffTools.ts

@@ -155,6 +155,7 @@ export function generateDbPairingId(db: DatabaseInfo): DatabaseInfo {
     procedures: db.procedures?.map(generateObjectPairingId),
     functions: db.functions?.map(generateObjectPairingId),
     triggers: db.triggers?.map(generateObjectPairingId),
+    schedulerEvents: db.schedulerEvents?.map(generateObjectPairingId),
     matviews: db.matviews?.map(generateObjectPairingId),
   };
 }
@@ -715,7 +716,15 @@ export function createAlterDatabasePlan(
 ): AlterPlan {
   const plan = new AlterPlan(wholeOldDb, wholeNewDb, driver.dialect, opts);
 
-  for (const objectTypeField of ['tables', 'views', 'procedures', 'matviews', 'functions']) {
+  for (const objectTypeField of [
+    'tables',
+    'views',
+    'procedures',
+    'matviews',
+    'functions',
+    'triggers',
+    'schedulerEvents',
+  ]) {
     for (const oldobj of oldDb[objectTypeField] || []) {
       const newobj = (newDb[objectTypeField] || []).find(x => x.pairingId == oldobj.pairingId);
       if (objectTypeField == 'tables') {

packages/tools/src/driverBase.ts

@@ -1,4 +1,5 @@
 import _compact from 'lodash/compact';
+import _isString from 'lodash/isString';
 import { SqlDumper } from './SqlDumper';
 import { splitQuery } from 'dbgate-query-splitter';
 import { dumpSqlSelect } from 'dbgate-sqltree';
@@ -26,9 +27,17 @@ const dialect = {
   defaultSchemaName: null,
 };
 
-export async function runCommandOnDriver(pool, driver: EngineDriver, cmd: (dmp: SqlDumper) => void): Promise<void> {
+export async function runCommandOnDriver(
+  pool,
+  driver: EngineDriver,
+  cmd: (dmp: SqlDumper) => void | string
+): Promise<void> {
   const dmp = driver.createDumper();
-  cmd(dmp as any);
+  if (_isString(cmd)) {
+    dmp.put(cmd);
+  } else {
+    cmd(dmp as any);
+  }
   // console.log('CMD:', dmp.s);
   await driver.query(pool, dmp.s, { discardResult: true });
 }
@@ -39,11 +48,21 @@ export async function runQueryOnDriver(
   cmd: (dmp: SqlDumper) => void
 ): Promise<QueryResult> {
   const dmp = driver.createDumper();
-  cmd(dmp as any);
+  if (_isString(cmd)) {
+    dmp.put(cmd);
+  } else {
+    cmd(dmp as any);
+  }
   // console.log('QUERY:', dmp.s);
   return await driver.query(pool, dmp.s);
 }
 
+export function formatQueryWithoutParams(driver: EngineDriver, sql: string) {
+  const dmp = driver.createDumper();
+  dmp.put(sql);
+  return dmp.s;
+}
+
 export const driverBase = {
   analyserClass: null,
   dumperClass: SqlDumper,

packages/tools/src/filterName.ts

@@ -2,9 +2,9 @@ import _compact from 'lodash/compact';
 import _isString from 'lodash/isString';
 import _startCase from 'lodash/startCase';
 
-export interface FilterNameDefinition {
-  childName: string;
-}
+// export interface FilterNameDefinition {
+//   childName: string;
+// }
 
 function camelMatch(filter: string, text: string): boolean {
   if (!text) return false;
@@ -20,7 +20,7 @@ function camelMatch(filter: string, text: string): boolean {
   }
 }
 
-export function filterName(filter: string, ...names: (string | FilterNameDefinition)[]) {
+export function filterName(filter: string, ...names: string[]) {
   if (!filter) return true;
 
   // const camelVariants = [name.replace(/[^A-Z]/g, '')]
@@ -28,22 +28,135 @@ export function filterName(filter: string, ...names: (string | FilterNameDefinit
 
   const namesCompacted = _compact(names);
 
-  // @ts-ignore
-  const namesOwn: string[] = namesCompacted.filter(x => _isString(x));
-  // @ts-ignore
-  const namesChild: string[] = namesCompacted.filter(x => x.childName).map(x => x.childName);
-
   for (const token of tokens) {
-    // const tokenUpper = token.toUpperCase();
-    if (token.startsWith('#')) {
-      // const tokenUpperSub = tokenUpper.substring(1);
-      const found = namesChild.find(name => camelMatch(token.substring(1), name));
-      if (!found) return false;
-    } else {
-      const found = namesOwn.find(name => camelMatch(token, name));
-      if (!found) return false;
-    }
+    const found = namesCompacted.find(name => camelMatch(token, name));
+    if (!found) return false;
   }
 
   return true;
 }
+
+export function filterNameCompoud(
+  filter: string,
+  namesMain: string[],
+  namesChild: string[]
+): 'main' | 'child' | 'both' | 'none' {
+  if (!filter) return 'both';
+
+  // const camelVariants = [name.replace(/[^A-Z]/g, '')]
+  const tokens = filter.split(' ').map(x => x.trim());
+
+  const namesCompactedMain = _compact(namesMain);
+  const namesCompactedChild = _compact(namesChild);
+
+  let isMainOnly = true;
+  let isChildOnly = true;
+
+  for (const token of tokens) {
+    const foundMain = namesCompactedMain.find(name => camelMatch(token, name));
+    const foundChild = namesCompactedChild.find(name => camelMatch(token, name));
+    if (!foundMain && !foundChild) return 'none';
+
+    if (!foundMain) isMainOnly = false;
+    if (!foundChild) isChildOnly = false;
+  }
+
+  if (isMainOnly && isChildOnly) return 'both';
+  if (isMainOnly) return 'main';
+  if (isChildOnly) return 'child';
+  return 'none';
+}
+
+export function tokenizeBySearchFilter(text: string, filter: string): { text: string; isMatch: boolean }[] {
+  const camelTokens = [];
+  const stdTokens = [];
+  for (const token of filter.split(' ').map(x => x.trim())) {
+    if (token.replace(/[A-Z]/g, '').length == 0) {
+      camelTokens.push(token);
+    } else {
+      stdTokens.push(token.toUpperCase());
+    }
+  }
+
+  let res = [
+    {
+      text,
+      isMatch: false,
+    },
+  ];
+
+  for (const token of camelTokens) {
+    const nextres = [];
+    for (const item of res) {
+      const indexes = [];
+      for (const char of token) {
+        if (indexes.length == 0 && char == item.text[0]?.toUpperCase()) {
+          // handle first letter of camelcase
+          indexes.push(0);
+        } else {
+          const index = item.text.indexOf(char, indexes.length > 0 ? indexes[indexes.length - 1] + 1 : 0);
+          if (index < 0) {
+            indexes.push(-1);
+          } else {
+            indexes.push(index);
+          }
+        }
+      }
+      if (indexes.some(x => x < 0)) {
+        nextres.push(item);
+      } else {
+        let lastIndex = 0;
+        for (let i = 0; i < indexes.length; i++) {
+          if (indexes[i] > lastIndex) {
+            nextres.push({ text: item.text.substring(lastIndex, indexes[i]), isMatch: false });
+          }
+          nextres.push({ text: item.text.substring(indexes[i], indexes[i] + 1), isMatch: true });
+          lastIndex = indexes[i] + 1;
+        }
+        nextres.push({ text: item.text.substring(lastIndex), isMatch: false });
+      }
+    }
+    res = nextres;
+  }
+
+  for (const token of stdTokens) {
+    const nextres = [];
+    for (const item of res) {
+      const index = item.text?.toUpperCase().indexOf(token);
+      if (index < 0) {
+        nextres.push(item);
+      } else {
+        nextres.push({ text: item.text.substring(0, index), isMatch: false });
+        nextres.push({ text: item.text.substring(index, index + token.length), isMatch: true });
+        nextres.push({ text: item.text.substring(index + token.length), isMatch: false });
+      }
+    }
+    res = nextres;
+  }
+
+  res = res.filter(x => x.text.length > 0);
+
+  if (res.length == 1 && !res[0].isMatch) {
+    return null;
+  }
+
+  return res;
+
+  // const result = [];
+  // let lastMatch = 0;
+  // for (const token of tokens) {
+  //   const index = text.indexOf(token, lastMatch);
+  //   if (index < 0) {
+  //     result.push({ token, isMatch: false });
+  //     continue;
+  //   }
+
+  //   result.push({ token: text.substring(lastMatch, index), isMatch: false });
+  //   result.push({ token: text.substring(index, index + token.length), isMatch: true });
+  //   lastMatch = index + token.length;
+  // }
+
+  // result.push({ token: text.substring(lastMatch), isMatch: false });
+
+  // return result;
+}

packages/tools/src/structureTools.ts

@@ -96,10 +96,20 @@ function fillDatabaseExtendedInfo(db: DatabaseInfo): DatabaseInfo {
     procedures: (db.procedures || []).map(obj => ({
       ...obj,
       objectTypeField: 'procedures',
+      parameters: (obj.parameters || []).map(param => ({
+        ...param,
+        pureName: obj.pureName,
+        schemaName: obj.schemaName,
+      })),
     })),
     functions: (db.functions || []).map(obj => ({
       ...obj,
       objectTypeField: 'functions',
+      parameters: (obj.parameters || []).map(param => ({
+        ...param,
+        pureName: obj.pureName,
+        schemaName: obj.schemaName,
+      })),
     })),
     triggers: (db.triggers || []).map(obj => ({
       ...obj,