v7.0.5-beta.1

MySQL FULLTEXT support #1305

.github/ISSUE_TEMPLATE/bug_report.md

@@ -1,12 +1,14 @@
 ---
 name: Bug report
-about: Create a report to help us improve DbGate
+about: Create a report to help us improve DbGate (in ENGLISH)
 title: 'BUG: Say something here'
 labels: ''
 assignees: ''
 
 ---
 
+Please keep communication in ENGLISH to reach more contributors.
+
 **Describe the bug**
 A clear and concise description of what the bug is.
 

.github/ISSUE_TEMPLATE/feature_request.md

@@ -1,12 +1,14 @@
 ---
 name: Feature request
-about: Suggest an idea for DbGate
+about: Suggest an idea for DbGate (in ENGLISH)
 title: 'FEAT: '
 labels: ''
 assignees: ''
 
 ---
 
+Please keep communication in ENGLISH to reach more contributors.
+
 **Is your feature request related to a problem? Please describe.**
 A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
 

.github/ISSUE_TEMPLATE/question.md

@@ -1,12 +1,14 @@
 ---
 name: Question
-about: Ask a question about how to do something
+about: Ask a question about how to do something (in ENGLISH)
 title: 'QUESTION: Summary of your question'
 labels: ''
 assignees: ''
 
 ---
 
+Please keep communication in ENGLISH to reach more contributors.
+
 **Details:**
 Details about your question
 

.github/workflows/build-app-beta.yaml

@@ -21,6 +21,10 @@ jobs:
           - windows-2022
           - ubuntu-22.04
     steps:
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+      - run: python -m pip install --upgrade pip setuptools
       - name: Install python 3.11 (MacOS)
         if: matrix.os == 'macos-14'
         run: |

.github/workflows/build-app-check.yaml

@@ -21,6 +21,10 @@ jobs:
           - windows-2022
           - ubuntu-22.04
     steps:
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+      - run: python -m pip install --upgrade pip setuptools
       - name: Install python 3.11 (MacOS)
         if: matrix.os == 'macos-14'
         run: |

.github/workflows/build-app-pro-beta.yaml

@@ -21,6 +21,10 @@ jobs:
           - windows-2022
           - ubuntu-22.04
     steps:
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+      - run: python -m pip install --upgrade pip setuptools
       - name: Install python 3.11 (MacOS)
         if: matrix.os == 'macos-14'
         run: |
@@ -43,7 +47,7 @@ jobs:
           repository: dbgate/dbgate-pro
           token: ${{ secrets.GH_TOKEN }}
           path: dbgate-pro
-          ref: ae1fcf6e61c6f7dfbb21005daa259c68e899a80a
+          ref: ddfa9029c9852277c384d4f680ad1b03a4e3e80c
       - name: Merge dbgate/dbgate-pro
         run: |
           mkdir ../dbgate-pro

.github/workflows/build-app-pro.yaml

@@ -21,6 +21,10 @@ jobs:
           - windows-2022
           - ubuntu-22.04
     steps:
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+      - run: python -m pip install --upgrade pip setuptools
       - name: Install python 3.11 (MacOS)
         if: matrix.os == 'macos-14'
         run: |
@@ -43,7 +47,7 @@ jobs:
           repository: dbgate/dbgate-pro
           token: ${{ secrets.GH_TOKEN }}
           path: dbgate-pro
-          ref: ae1fcf6e61c6f7dfbb21005daa259c68e899a80a
+          ref: ddfa9029c9852277c384d4f680ad1b03a4e3e80c
       - name: Merge dbgate/dbgate-pro
         run: |
           mkdir ../dbgate-pro

.github/workflows/build-app.yaml

@@ -21,6 +21,10 @@ jobs:
           - windows-2022
           - ubuntu-22.04
     steps:
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+      - run: python -m pip install --upgrade pip setuptools
       - name: Install python 3.11 (MacOS)
         if: matrix.os == 'macos-14'
         run: |

.github/workflows/build-cloud-pro.yaml

@@ -39,7 +39,7 @@ jobs:
           repository: dbgate/dbgate-pro
           token: ${{ secrets.GH_TOKEN }}
           path: dbgate-pro
-          ref: ae1fcf6e61c6f7dfbb21005daa259c68e899a80a
+          ref: ddfa9029c9852277c384d4f680ad1b03a4e3e80c
       - name: Merge dbgate/dbgate-pro
         run: |
           mkdir ../dbgate-pro

.github/workflows/build-docker-pro.yaml

@@ -44,7 +44,7 @@ jobs:
           repository: dbgate/dbgate-pro
           token: ${{ secrets.GH_TOKEN }}
           path: dbgate-pro
-          ref: ae1fcf6e61c6f7dfbb21005daa259c68e899a80a
+          ref: ddfa9029c9852277c384d4f680ad1b03a4e3e80c
       - name: Merge dbgate/dbgate-pro
         run: |
           mkdir ../dbgate-pro

.github/workflows/build-npm-pro.yaml

@@ -35,7 +35,7 @@ jobs:
           repository: dbgate/dbgate-pro
           token: ${{ secrets.GH_TOKEN }}
           path: dbgate-pro
-          ref: ae1fcf6e61c6f7dfbb21005daa259c68e899a80a
+          ref: ddfa9029c9852277c384d4f680ad1b03a4e3e80c
       - name: Merge dbgate/dbgate-pro
         run: |
           mkdir ../dbgate-pro

.github/workflows/e2e-pro.yaml

@@ -5,10 +5,14 @@ name: Cypress tests with screenshots PREMIUM
 'on':
   push:
     branches:
+      - stable
       - master
       - develop
       - feature/**
       - hotfix/**
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
 jobs:
   e2e-tests:
     runs-on: ubuntu-latest
@@ -26,7 +30,7 @@ jobs:
           repository: dbgate/dbgate-pro
           token: ${{ secrets.GH_TOKEN }}
           path: dbgate-pro
-          ref: ae1fcf6e61c6f7dfbb21005daa259c68e899a80a
+          ref: ddfa9029c9852277c384d4f680ad1b03a4e3e80c
       - name: Merge dbgate/dbgate-pro
         run: |
           mkdir ../dbgate-pro
@@ -69,8 +73,8 @@ jobs:
         with:
           name: screenshots
           path: screenshots
-      - name: Push E2E screenshots
-        if: ${{ github.ref_name == 'master' }}
+      - name: Push E2E screenshots - stable
+        if: ${{ github.ref_name == 'stable' }}
         run: |
           git config --global user.email "info@dbgate.info"
           git config --global user.name "GitHub Actions"
@@ -80,6 +84,17 @@ jobs:
           git add .
           git commit --amend --no-edit
           git push --force
+      - name: Push E2E screenshots - master
+        if: ${{ github.ref_name == 'master' }}
+        run: |
+          git config --global user.email "info@dbgate.info"
+          git config --global user.name "GitHub Actions"
+          git clone https://${{ secrets.DIFLOW_GIT_SECRET }}@github.com/dbgate/dbgate-img.git
+          cp ../dbgate-merged/e2e-tests/screenshots/*.png dbgate-img/static/img-dev
+          cd dbgate-img/static/img-dev
+          git add .
+          git commit --amend --no-edit
+          git push --force
     services:
       postgres-cypress:
         image: postgres

.github/workflows/run-tests.yaml

@@ -9,6 +9,9 @@ name: Integration and unit tests
       - develop
       - feature/**
       - hotfix/**
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
 jobs:
   all-tests:
     runs-on: ubuntu-latest

CHANGELOG.md

@@ -8,7 +8,84 @@ Builds:
  - linux - application for linux
  - win - application for Windows
 
-## 6.7.2 - not released
+## 7.0.4
+- FIXED: MS SQL server export to CSV does not convert bit FALSE to 0 #1276
+- ADDED: MySQL FULLTEXT support #1305
+- FIXED: Error messages in Chinese will display garbled characters(MS SQL over ODBC) #1321
+- FIXED: Table's Show SQL fails to display precision and scale for NUMERIC/DECIMAL types in PostgreSQL #1325
+- FIXED: Export to Excel/CSV is broken for certain data types in v7.0.0 #1327
+- ADDED: Null value with keyboard shortcut in form view #1332
+- FIXED: Clicking into active form cell discards changes #1334
+- FIXED: Remember selection after filtering #1335
+- FIXED: Unable to use 'Group By' or one of the aggregate functions on tables containing text columns #1348
+- CHANGED: Improved custom connection color palette
+
+## 7.0.3
+- FIXED: Optimalized loading MySQL primary keys #1261
+- FIXED: Test connection now works for MS Entra authentication #1315
+- FIXED: SQL Server - Unable to use 'Is Empty or Null' or 'Has Not Empty Value' filters on a field with data type TEXT #1338
+- FIXED: Play triangle too large for text-wrapped queries #1337
+- FIXED: Text wraps mid-word in form view, making it illegible #1333
+- FIXED: Cell View autodetects Form instead of Map for geometry/geography #1330
+- FIXED: Search for database in cloud connection #1329
+- ADDED: Toolstrip could be configured to the bottom of the tab #1326
+- CHANGED: Upgraded node for DbGate AWS distribution
+
+## 7.0.1
+- FIXED: Foreign key actions not detected on PostgreSQL #1323
+- FIXED: Vulnerabilities in bundled dependencies: axios, cross-spawn, glob #1322
+- FIXED: The JsonB field in the cell data view always displays as null. #1320
+- ADDED: Possibility to skip computed coumn in SQL generator
+- ADDED: Improved team file editing, move between team folders
+- ADDED: Korean localization
+- FIXED: Added missing localization strings
+- ADDED: Default editor theme is part of application theme now
+
+## 7.0.0
+- CHANGED: New design of application, new theme system
+- ADDED: Theme AI assistant - create custom themes using AI (Premium)
+- CHANGED: Themes are now defined in JSON files, custom themes could be shared via DbGate Cloud
+- REMOVED: Custom themes are no longer part of plugins
+- CHANGED: Huge improvements of Redis support
+- ADDED: Support for Redis JSON and Stream types
+- ADDED: Editing Redis values (Strings, Hashes, Lists, Sets, Sorted Sets, JSON, Streams)
+- ADDED: Support for Team Folders (Team Premium)
+- CHANGED: Upgraded Svelte to version 4
+- ADDED: Differentiate pinned database with same name #1306
+- ADDED: Database icons/logos for faster visual recognition #1222
+- CHANGED: Reorganized left sidebar widgets
+- ADDED: Widget for currently opened tabs
+
+## 6.8.2
+- FIXED: Initialize storage database from envoronment variables failed with PostgreSQL
+
+## 6.8.1
+- FIXED: Won't navigate to the relevant field on click of a field in columns #1303
+
+## 6.8.0
+- ADDED: Form cell view for detailed data inspection and editing in data grids, with multi-row bulk editing support
+- CHANGED: Cell data sidebar moved to right side, now is part of data grid
+- FIXED: Improved widget resizing algorithm
+- FIXED: Word wrap feature in SQL editor
+- CHANGED: Data grid keyboard navigation improvements
+- CHANGED: Improved PostgreSQL decimal type support in data grid #1214
+- ADDED: Retrieve number of databases from Redis configuration #1278
+- ADDED: Run macro context menu (Premium)
+- ADDED: Support for skip update columns in replicator
+- FIXED: UTF-8 BOM handling in CSV input
+- CHANGED: Advanced export is now part of Community edition
+- FIXED: SQLite foreign key constraint types
+- FIXED: Double drop constraint issue
+- CHANGED: Improved map view lat/lon field autodetection
+- FIXED: Alter table operations and constraint sanitization
+- ADDED: Import connections from environment variables (Team Premium)
+
+## 6.7.3
+- FIXED: Fixed problem in analyser core - in PostgreSQL, after dropping table, dropped table still appeared in structure
+- FIXED: PostgreSQL numeric columns do not align right #1254
+- ADDED: Custom thousands separator #1213
+
+## 6.7.2
 - CHANGED: Settings modal redesign - now is settings opened in tab instead of modal, similarily as in VSCode
 - FIXED: Fixed search in table shortcuts #1273
 - CHANGED: Improved foreign key editor UX
@@ -74,7 +151,7 @@ Builds:
 - ADDED: SQL AI assistant - powered by database chat, could help you to write SQL queries (Premium)
 - ADDED: Explain SQL error (powered by AI) (Premium)
 - ADDED: Database chat (and SQL AI Assistant) now supports showing charts (Premium)
-- FIXED: Fxied editing new files and roles (Team Premium)
+- FIXED: Fixed editing new files and roles (Team Premium)
 - FIXED: Connection to standalone database could be now pinned
 - FIXED: Cannot open up large JSON file #1215
 

README.md

@@ -61,7 +61,7 @@ DbGate is licensed under GPL-3.0 license and is free to use for any purpose.
 * Edit table schema, indexes, primary and foreign keys
 * Compare and synchronize database structure
 * ER diagram
-* Light and dark theme, next themes available as plugins from github community
+* Light and dark theme, next themes available from DbGate Cloud
 * Huge support for work with related data - master/detail views, foreign key lookups, expanding columns from related tables in flat data view
 * Query designer - visual SQL query builder without writing SQL code. Complex conditions like WHERE NOT EXISTS.
 * Query perspectives – innovative nested table view over complex relational data, something like query designer on MongoDB databases
@@ -94,7 +94,8 @@ Any contributions are welcome. If you want to contribute without coding, conside
 * Create some tutorial video on [youtube](https://www.youtube.com/playlist?list=PLCo7KjCVXhr0RfUSjM9wJMsp_ShL1q61A)
 * Become a backer on [GitHub sponsors](https://github.com/sponsors/dbgate) or [Open collective](https://opencollective.com/dbgate)
 * Add a SQL script to [Public Knowledge Base](https://github.com/dbgate/dbgate-knowledge-base)
-* Where a small coding is acceptable for you, you could [create plugin](https://docs.dbgate.io/plugin-development). Plugins for new themes can be created actually without JS coding
+* Where a small coding is acceptable for you, you could [create plugin](https://docs.dbgate.io/plugin-development)
+* Create a new custom theme and share it on [DbGate Cloud](https://github.com/dbgate/dbgate-knowledge-base/tree/master/folder-Themes)
 
 Thank you!
 

app/package.json

@@ -1,6 +1,6 @@
 {
   "name": "dbgate",
-  "version": "6.0.0-alpha.1",
+  "version": "7.0.0-alpha.1",
   "private": true,
   "author": "Jan Prochazka <jenasoft.database@gmail.com>",
   "description": "Opensource database administration tool",

common/translations-cli/translate.js

@@ -15,6 +15,7 @@ const languageNames = {
     'fr.json': 'French',
     'it.json': 'Italian',
     'ja.json': 'Japanese',
+    'ko.json': 'Korean',
     'pt.json': 'Portuguese',
     'sk.json': 'Slovak',
     'zh.json': 'Chinese'

e2e-tests/cypress.config.js

@@ -10,6 +10,7 @@ module.exports = defineConfig({
     // baseUrl: 'http://localhost:3000',
     // trashAssetsBeforeRuns: false,
     chromeWebSecurity: false,
+    reporter: process.env.CI ? 'mocha-reporter-gha' : 'spec',
 
     setupNodeEvents(on, config) {
       // implement node event listeners here
@@ -48,6 +49,9 @@ module.exports = defineConfig({
             case 'charts':
               serverProcess = exec('yarn start:charts');
               break;
+            case 'redis':
+              serverProcess = exec('yarn start:redis');
+              break;
           }
 
           await waitOn({ resources: ['http://localhost:3000'] });

e2e-tests/cypress/e2e/browse-data.cy.js

@@ -85,14 +85,16 @@ describe('Data browser data', () => {
     cy.contains('MySql-connection').click();
     cy.contains('MyChinook').click();
     cy.contains('Album').click();
+
+    // hide what is not needed
+    cy.testid('WidgetIconPanel_database').click();
+    cy.testid('DataGrid_itemReferences').click();
+
     cy.testid('DataFilterControl_input_Title').type('Rock{enter}');
     cy.contains('Rows: 7');
     cy.testid('DataFilterControl_input_AlbumId').type('>10xxx{enter}');
     cy.contains('Rows: 7');
-    cy.testid('DataFilterControl_filtermenu_Title').click();
-    // hide what is not needed
-    cy.testid('WidgetIconPanel_database').click();
-    cy.testid('DataGrid_itemReferences').click();
+    cy.testid('DataFilterControl_filtermenu_ArtistId').click();
     cy.themeshot('data-browser-filter');
     cy.testid('DataGridCore_button_clearFilters').click();
     cy.contains('Rows: 347');
@@ -202,7 +204,7 @@ describe('Data browser data', () => {
     cy.themeshot('query-editor-join-wizard');
   });
 
-  it('Mongo JSON data view', () => {
+  it('Mongo query JSON data view', () => {
     cy.contains('Mongo-connection').click();
     cy.contains('MgChinook').click();
     cy.contains('Customer').click();
@@ -213,9 +215,10 @@ describe('Data browser data', () => {
     cy.contains('Open query').click();
     cy.wait(1000);
     cy.contains('Execute').click();
-    cy.testid('WidgetIconPanel_cell-data').click();
+    cy.testid('TabContent_1').contains('Leonie').rightclick();
+    cy.contains('Show cell data').click();
     // test JSON view
-    cy.contains('Country: "Brazil"');
+    cy.contains('Country: "Germany"');
     cy.themeshot('mongo-query-json-view');
   });
 
@@ -293,7 +296,8 @@ describe('Data browser data', () => {
     // cy.contains('location').click();
     cy.contains('14.2').click();
     cy.contains('13.9').click({ shiftKey: true });
-    cy.testid('WidgetIconPanel_cell-data').click();
+    cy.testid('WidgetIconPanel_database').click();
+    cy.testid('TableDataTab_toggleCellDataView').click();
     cy.wait(2000);
     cy.themeshot('cell-map-view');
   });
@@ -337,7 +341,7 @@ describe('Data browser data', () => {
     cy.themeshot('save-changes-mongodb');
   });
 
-  it('Edit mongo data JSON', () => {
+  it('Mongo JSON cell view', () => {
     // TODO FIX: Auto expand cell view
     cy.contains('Mongo-connection').click();
     cy.contains('MgRivers').click();
@@ -347,7 +351,8 @@ describe('Data browser data', () => {
     cy.testid('ColumnManagerRow_checkbox_countries.1').click();
     cy.testid('ColumnManagerRow_checkbox__id').click();
     cy.testid('DataFilterControl_input_countries.1').type('EXISTS{enter}');
-    cy.testid('WidgetIconPanel_cell-data').click();
+    cy.contains('Austria').click();
+    cy.testid('CollectionDataTab_toggleCellDataView').click();
     cy.themeshot('mongodb-json-cell-view');
   });
 
@@ -472,4 +477,39 @@ describe('Data browser data', () => {
     cy.testid('DataDeployTab_importIntoDb').click();
     cy.themeshot('data-replicator');
   });
+
+  it('Form cell view', () => {
+    cy.contains('MySql-connection').click();
+    cy.contains('MyChinook').click();
+    cy.contains('Invoice').click();
+    cy.contains('Rows: 412');
+    cy.get('[data-row="0"][data-col="header"]').click();
+    cy.get('[data-row="1"][data-col="header"]').click();
+    cy.get('[data-row="0"][data-col="header"]').click();
+    cy.contains('Autodetect - Form');
+    cy.themeshot('form-cell-view');
+  });
+
+  it('Group by', () => {
+    cy.contains('MySql-connection').click();
+    cy.contains('MyChinook').click();
+    cy.contains('Album').click();
+    cy.testid('WidgetIconPanel_database').click();
+    cy.testid('ColumnHeaderControl_dropdown_ArtistId').click();
+    cy.contains('Group by').click();
+    cy.testid('ColumnHeaderControl_dropdown_Title').first().click();
+    cy.themeshot('data-browser-group-by');
+  });
+
+  it('Filter by expanded column', () => {
+    cy.contains('MySql-connection').click();
+    cy.contains('MyChinook').click();
+    cy.contains('Album').click();
+    cy.testid('WidgetIconPanel_database').click();
+    cy.testid('ColumnManagerRow_expand_ArtistId').click();
+    cy.testid('ColumnManagerRow_checkbox_ArtistId.Name').click();
+    cy.testid('ColumnManagerRow_checkbox_ArtistId').click();
+    cy.testid('DataFilterControl_input_ArtistId.Name').type('mich{enter}');
+    cy.themeshot('data-browser-filter-by-expanded');
+  });
 });

e2e-tests/cypress/e2e/charts.cy.js

@@ -110,7 +110,7 @@ describe('Charts', () => {
     cy.themeshot('new-object-window');
   });
 
-  it('Database chat - charts', () => {
+  it.skip('Database chat - charts', () => {
     cy.contains('MySql-connection').click();
     cy.contains('MyChinook').click();
     cy.testid('TabsPanel_buttonNewObject').click();
@@ -125,7 +125,7 @@ describe('Charts', () => {
     cy.themeshot('database-chat-chart');
   });
 
-  it('Database chat', () => {
+  it.skip('Database chat', () => {
     cy.contains('MySql-connection').click();
     cy.contains('MyChinook').click();
     cy.testid('TabsPanel_buttonNewObject').click();
@@ -146,7 +146,7 @@ describe('Charts', () => {
     // cy.themeshot('database-chat');
   });
 
-  it('Explain query error', () => {
+  it.skip('Explain query error', () => {
     cy.contains('MySql-connection').click();
     cy.contains('MyChinook').click();
     cy.testid('TabsPanel_buttonNewObject').click();
@@ -225,14 +225,11 @@ describe('Charts', () => {
     cy.contains('Default Actions').click();
     cy.get('[data-testid=DefaultActionsSettings_useLastUsedAction]').uncheck();
 
-
     // Themes
     cy.contains('Themes').click();
     cy.themeshot('app-settings-themes');
-    cy.contains('Dark').click();
-    cy.get('body').find('.theme-dark').should('exist');
-    cy.contains('Light').click();
-    cy.get('body').find('.theme-light').should('exist');
+    cy.testid('ThemeSkeleton-Dark-built-in').click();
+    cy.testid('ThemeSkeleton-Light-built-in').click();
 
     // General
     cy.contains(/^General$/).click();
@@ -258,7 +255,6 @@ describe('Charts', () => {
     cy.contains('OK').click();
     cy.contains('Ctrl+G');
 
-    
     cy.contains('AI').click();
     cy.themeshot('app-settings-ai');
     cy.get('[data-testid=AISettings_addProviderButton]').click();
@@ -268,4 +264,22 @@ describe('Charts', () => {
     cy.contains('OK').click();
     cy.contains('Provider 1').should('not.exist');
   });
+
+  it('Custom theme', () => {
+    cy.testid('WidgetIconPanel_settings').click();
+    cy.contains('Themes').click();
+    cy.testid('ThemeSettings-themeList').contains('Green-Sample').click();
+    cy.testid('WidgetIconPanel_file').click();
+    cy.themeshot('green-theme', { keepTheme: true });
+
+    cy.testid('ThemeSettings-themeList').contains('Solarized-light').click();
+    cy.testid('WidgetIconPanel_database').click();
+    cy.contains('MySql-connection').click();
+    cy.contains('MyChinook').click();
+    cy.contains('Customer').click();
+    cy.contains('Leonie');
+    cy.testid('WidgetIconPanel_file').click();
+
+    cy.themeshot('solarized-theme', { keepTheme: true });
+  });
 });

e2e-tests/cypress/e2e/multi-sql.cy.js

@@ -3,6 +3,8 @@ const { formatQueryWithoutParams } = require('dbgate-tools');
 
 global.DBGATE_PACKAGES = {
   'dbgate-tools': require('dbgate-tools'),
+  'dbgate-sqltree': require('dbgate-sqltree'),
+  'dbgate-datalib': require('dbgate-datalib'),
 };
 
 function requireEngineDriver(engine) {
@@ -141,7 +143,7 @@ describe('Backup table', () => {
     cy.get('body').realType('111222333{enter}');
 
     cy.testid('TableDataTab_save').click();
-    cy.testid('ConfirmSqlModal_okButton').click();
+    cy.testid('ConfirmSqlModal_okButton', { timeout: 10000 }).click();
     cy.contains('Rows: 11').should('be.visible'); // wait for save
 
     cy.testid('app-object-group-items-table-backups').contains('addresses').rightclick();
@@ -161,7 +163,7 @@ describe('Backup table', () => {
     // cy.testid('CloseTabModal_buttonConfirm').click();
     cy.wait(1000);
 
-    cy.testid('app-object-group-items-tables').contains('addresses').click();
+    cy.testid('app-object-group-items-tables').contains('addresses', { timeout: 10000 }).click();
 
     // check whether data was successfully restored
     cy.contains('Rows: 12').should('be.visible');
@@ -237,7 +239,7 @@ describe('Import CSV - source error', () => {
     cy.testid('ImportExportTab_preview_content').contains('Invalid Closing Quote').should('be.visible');
 
     cy.testid('ImportExportTab_executeButton').click();
-    cy.testid('ImportExportConfigurator_errorInfoIcon_customers-20-err').click();
+    cy.testid('ImportExportConfigurator_errorInfoIcon_customers-20-err', { timeout: 10000 }).click();
 
     cy.testid('ErrorMessageModal_message').contains('Invalid Closing Quote').should('be.visible');
   });
@@ -256,7 +258,7 @@ describe('Import CSV - target error', () => {
     cy.contains('customers-20');
     cy.testid('ImportExportConfigurator_targetName_customers-20').clear().type('system."]`');
     cy.testid('ImportExportTab_executeButton').click();
-    cy.testid('ImportExportConfigurator_errorInfoIcon_customers-20').click();
+    cy.testid('ImportExportConfigurator_errorInfoIcon_customers-20', { timeout: 10000 }).click();
     cy.testid('ErrorMessageModal_message').should('be.visible');
   });
 });

e2e-tests/cypress/e2e/redis.cy.js

@@ -0,0 +1,120 @@
+Cypress.on('uncaught:exception', (err, runnable) => {
+  // if the error message matches the one about WorkerGlobalScope importScripts
+  if (err.message.includes("Failed to execute 'importScripts' on 'WorkerGlobalScope'")) {
+    // return false to let Cypress know we intentionally want to ignore this error
+    return false;
+  }
+  // otherwise let Cypress throw the error
+});
+
+beforeEach(() => {
+  cy.visit('http://localhost:3000');
+  cy.viewport(1250, 900);
+});
+
+describe('Redis data', () => {
+  it('String test', () => {
+    cy.contains('Redis-connection').click();
+    cy.contains('db1').click();
+    cy.contains('app').click();
+    cy.contains('version').click();
+    cy.testid('RedisValueDetail_AceEditor').click().realPress('Backspace').realType('1');
+    cy.contains('Save').click();
+    cy.contains('OK').click();
+  });
+
+  it('Hash test', () => {
+    cy.contains('Redis-connection').click();
+    cy.contains('db1').click();
+    cy.contains('user').click();
+    cy.contains('alice').click();
+    cy.testid('RedisKeyDetailTab_RenameKeyButton').click();
+    cy.themeshot('redis-rename-key');
+    cy.realType('3');
+    cy.contains('OK').click();
+    cy.contains('age').click();
+    cy.testid('RedisValueHashDetail_ValueSection').click().realPress('Backspace').realType('8');
+    cy.contains('Add field').click();
+    cy.testid('RedisValueListLikeEdit_key').click().realType('phone');
+    cy.testid('RedisValueListLikeEdit_value').click().realType('123-456-7890');
+    cy.contains('Refresh').click();
+    cy.themeshot('redis-hash-edit');
+    cy.contains('Save').click();
+    cy.themeshot('redis-hash-script-edit');
+    cy.contains('OK').click();
+  });
+
+  it('List test', () => {
+    cy.contains('Redis-connection').click();
+    cy.contains('db1').click();
+    cy.contains('queue').click();
+    cy.contains('emails').click();
+    cy.contains('Add field').click();
+    cy.testid('RedisValueListLikeEdit_value').click().realType('reset');
+    cy.contains('Save').click();
+    cy.contains('OK').click();
+  });
+
+  it('Set test', () => {
+    cy.contains('Redis-connection').click();
+    cy.contains('db1').click();
+    cy.contains('tags').click();
+    cy.contains('Add field').click();
+    cy.testid('RedisValueListLikeEdit_value').click().realType('newtag');
+    cy.contains('Save').click();
+    cy.contains('OK').click();
+  });
+
+  it('ZSet test', () => {
+    cy.contains('Redis-connection').click();
+    cy.contains('db1').click();
+    cy.contains('leaderboard').click();
+    cy.contains('alice').click();
+    cy.testid('RedisValueZSetDetail_score')
+      .click()
+      .realPress('Backspace')
+      .realPress('Backspace')
+      .realPress('Backspace')
+      .realType('35');
+    cy.contains('Save').click();
+    cy.contains('OK').click();
+    cy.contains('35').should('exist');
+  });
+
+  it('JSON test', () => {
+    cy.contains('Redis-connection').click();
+    cy.contains('db1').click();
+    cy.contains('user').click();
+    cy.contains('1:*').click();
+    cy.contains('json').click();
+    cy.testid('RedisValueDetail_displaySelect').select('JSON view');
+    cy.themeshot('redis-json-detail');
+  });
+
+  it('Stream test', () => {
+    cy.contains('Redis-connection').click();
+    cy.contains('db1').click();
+    cy.contains('events').click();
+    cy.contains('Add field').click();
+    cy.testid('RedisValueListLikeEdit_field').click().realType('message');
+    cy.testid('RedisValueListLikeEdit_value').click().realType('Hello, World!');
+    cy.contains('Save').click();
+    cy.contains('OK').click();
+    cy.themeshot('redis-stream');
+  });
+
+  it('Add key', () => {
+    cy.contains('Redis-connection').click();
+    cy.contains('db1').click();
+    cy.testid('RedisKeysTree_addKeyDropdown').click();
+    cy.contains('String').click();
+    cy.testid('NewRedisKeyTab_keyName').click().realType('newstringkey');
+    cy.testid('RedisValueDetail_AceEditor').click().realType('This is a new string key.');
+    cy.contains('Save').click();
+    cy.contains('OK').click();
+    cy.contains('newstringkey').should('exist');
+    cy.testid('RedisKeysTree_addKeyDropdown').click();
+    cy.contains('Hash').click();
+    cy.themeshot('redis-add-hash-key');
+  });
+});

e2e-tests/cypress/support/commands.js

@@ -36,9 +36,11 @@ Cypress.Commands.add(
     prevSubject: 'optional',
   },
   (subject, file, options) => {
-    cy.window().then(win => {
-      win.__changeCurrentTheme('theme-dark');
-    });
+    if (!options?.keepTheme) {
+      cy.window().then(win => {
+        win.__changeCurrentTheme('dark');
+      });
+    }
 
     // cy.screenshot(`${file}-dark`, {
     //   onAfterScreenshot: (doc, props) => {
@@ -63,9 +65,11 @@ Cypress.Commands.add(
     //   });
     // });
 
-    cy.window().then(win => {
-      win.__changeCurrentTheme('theme-light');
-    });
+    if (!options?.keepTheme) {
+      cy.window().then(win => {
+        win.__changeCurrentTheme('light');
+      });
+    }
 
     if (subject) {
       cy.wrap(subject).screenshot(`${file}-light`, options);

e2e-tests/data/files/themes/Green-Sample

@@ -0,0 +1,253 @@
+{
+  "themeName": "Green-Sample",
+  "themeType": "light",
+  "themeVariables": {
+    "--theme-generic-font": "oklch(27% 0.07 130)",
+    "--theme-generic-font-hover": "oklch(40% 0.15 130)",
+    "--theme-generic-font-grayed": "oklch(65% 0.05 130)",
+    "--theme-link-foreground": "oklch(40% 0.25 130)",
+    "--theme-content-background": "oklch(95% 0.05 130)",
+    "--theme-widget-panel-background": "oklch(80% 0.1 130)",
+    "--theme-widget-panel-foreground": "oklch(27% 0.07 130)",
+    "--theme-widget-icon-background-active": "oklch(50% 0.12 130)",
+    "--theme-widget-icon-foreground-active": "white",
+    "--theme-widget-icon-foreground-hover": "white",
+    "--theme-widget-icon-border-active": "1px solid white",
+    "--theme-scrollbar-background": "oklch(90% 0.08 130)",
+    "--theme-scrollbar-thumb-background": "oklch(70% 0.12 130)",
+    "--theme-scrollbar-thumb-background-hover": "oklch(40% 0.15 130)",
+    "--theme-scrollbar-corner-background": "oklch(85% 0.1 130)",
+    "--theme-tabs-panel-border": "1px solid oklch(95% 0.05 130)",
+    "--theme-tabs-panel-foreground": "oklch(20% 0.06 130)",
+    "--theme-tabs-panel-active-foreground": "oklch(10% 0.06 130)",
+    "--theme-tabs-panel-background": "oklch(95.5% 0.04 130)",
+    "--theme-tabs-panel-active-background": "oklch(80% 0.12 130)",
+    "--theme-tabs-panel-item-background": "oklch(90% 0.1 130)",
+    "--theme-tabs-panel-active-border": "1px solid oklch(50% 0.2 130)",
+    "--theme-splitter-active": "oklch(50% 0.2 130)",
+    "--theme-splitter-button-background": "oklch(90% 0.1 130)",
+    "--theme-splitter-button-background-active": "oklch(85% 0.15 130)",
+    "--theme-splitter-button-foreground": "oklch(10% 0.06 130)",
+    "--theme-sidebar-background": "oklch(90% 0.1 130)",
+    "--theme-sidebar-background-hover": "oklch(80% 0.12 130)",
+    "--theme-sidebar-background-active": "oklch(75% 0.14 130)",
+    "--theme-sidebar-background-focused": "oklch(70% 0.18 130)",
+    "--theme-sidebar-foreground": "oklch(20% 0.06 130)",
+    "--theme-sidebar-foreground-button": "oklch(40% 0.12 130)",
+    "--theme-sidebar-foreground-grayed": "oklch(65% 0.05 130)",
+    "--theme-sidebar-foreground-hover": "oklch(50% 0.25 130)",
+    "--theme-sidebar-section-background": "oklch(65% 0.05 130)",
+    "--theme-sidebar-section-border": "none",
+    "--theme-sidebar-section-border-top": "1px solid oklch(80% 0.1 130)",
+    "--theme-sidebar-section-foreground": "oklch(10% 0.06 130)",
+    "--theme-sidebar-border": "none",
+    "--theme-altsidebar-background": "oklch(95% 0.05 130)",
+    "--theme-altsidebar-background-grayed": "oklch(97% 0.02 130)",
+    "--theme-altsidebar-background-hover": "oklch(85% 0.1 130)",
+    "--theme-altsidebar-background-active": "oklch(80% 0.12 130)",
+    "--theme-altsidebar-background-focused": "oklch(75% 0.15 130)",
+    "--theme-altsidebar-foreground": "oklch(20% 0.06 130)",
+    "--theme-altsidebar-foreground-button": "oklch(40% 0.12 130)",
+    "--theme-altsidebar-foreground-grayed": "oklch(65% 0.05 130)",
+    "--theme-altsidebar-foreground-hover": "oklch(50% 0.25 130)",
+    "--theme-altsidebar-section-background": "oklch(97% 0.02 130)",
+    "--theme-altsidebar-section-border": "none",
+    "--theme-altsidebar-section-border-top": "1px solid oklch(85% 0.1 130)",
+    "--theme-altsidebar-section-foreground": "oklch(10% 0.06 130)",
+    "--theme-altsidebar-border": "1px solid oklch(90% 0.1 130)",
+    "--theme-searchbox-background": "oklch(80% 0.12 130)",
+    "--theme-searchbox-placeholder": "oklch(65% 0.05 130)",
+    "--theme-searchbox-border": "1px solid oklch(70% 0.15 130)",
+    "--theme-searchbox-background-filtered": "oklch(95% 0.04 110)",
+    "--theme-altsearchbox-background": "oklch(90% 0.1 130)",
+    "--theme-altsearchbox-placeholder": "oklch(65% 0.05 130)",
+    "--theme-altsearchbox-border": "1px solid oklch(80% 0.1 130)",
+    "--theme-inlinebutton-foreground": "oklch(40% 0.12 130)",
+    "--theme-inlinebutton-foreground-disabled": "oklch(65% 0.05 130)",
+    "--theme-inlinebutton-foreground-hover": "black",
+    "--theme-inlinebutton-circle-hover-background": "oklch(85% 0.1 130)",
+    "--theme-inlinebutton-bordered-border": "1px solid oklch(85% 0.1 130)",
+    "--theme-inlinebutton-bordered-hover-border": "1px solid oklch(70% 0.15 130)",
+    "--theme-inlinebutton-bordered-background": "linear-gradient(to bottom, oklch(95% 0.04 130) 5%, oklch(90% 0.1 130) 100%)",
+    "--theme-inlinebutton-bordered-hover-background": "linear-gradient(to bottom, oklch(90% 0.1 130) 5%, oklch(95% 0.04 130) 100%)",
+    "--theme-datagrid-background": "oklch(95% 0.04 130)",
+    "--theme-datagrid-foreground": "oklch(20% 0.06 130)",
+    "--theme-datagrid-foreground-grayed": "oklch(65% 0.05 130)",
+    "--theme-datagrid-border-horizontal": "1px solid oklch(90% 0.1 130)",
+    "--theme-datagrid-border-vertical": "1px solid oklch(95% 0.04 130)",
+    "--theme-datagrid-cell-background": "oklch(97% 0.02 130)",
+    "--theme-datagrid-headercell-background": "oklch(95% 0.04 130)",
+    "--theme-datagrid-cell-background-alt": "oklch(95% 0.04 130)",
+    "--theme-datagrid-cell-background-alt2": "oklch(90% 0.1 130)",
+    "--theme-datagrid-filter-background": "oklch(90% 0.1 130)",
+    "--theme-datagrid-filter-border": "1px solid oklch(85% 0.1 130)",
+    "--theme-datagrid-filter-ok-background": "oklch(95% 0.1 135)",
+    "--theme-datagrid-filter-error-background": "oklch(95% 0.12 30)",
+    "--theme-datagrid-modified-row-background": "oklch(95% 0.1 135)",
+    "--theme-datagrid-modified-cell-background": "oklch(90% 0.15 135)",
+    "--theme-datagrid-inserted-row-background": "oklch(95% 0.1 110)",
+    "--theme-datagrid-deleted-row-background": "oklch(95% 0.1 25)",
+    "--theme-datagrid-selected-cell-background": "oklch(80% 0.1 130)",
+    "--theme-datagrid-focused-cell-background": "oklch(75% 0.15 130)",
+    "--theme-datagrid-focused-cell-border-horizontal": "1px solid oklch(70% 0.2 130)",
+    "--theme-datagrid-focused-cell-border-vertical": "1px solid oklch(70% 0.2 130)",
+    "--theme-datagrid-selected-point-marker": "oklch(50% 0.25 130)",
+    "--theme-datagrid-corner-label-background": "oklch(75% 0.15 130)",
+    "--theme-datagrid-corner-label-border": "1px solid oklch(70% 0.2 130)",
+    "--theme-datagrid-detail-header-background": "oklch(85% 0.05 130)",
+    "--theme-datagrid-detail-header-border": "1px solid oklch(80% 0.1 130)",
+    "--theme-datagrid-cell-foreground-value-green": "oklch(45% 0.2 140)",
+    "--theme-checkbox-check": "oklch(90% 0.1 130)",
+    "--theme-checkbox-background": "oklch(40% 0.25 130)",
+    "--theme-checkbox-border": "1px solid oklch(70% 0.15 130)",
+    "--theme-checkbox-mark": "white",
+    "--theme-checkbox-background-disabled": "oklch(95% 0.04 130)",
+    "--theme-checkbox-background-disabled-before": "oklch(70% 0.15 130)",
+    "--theme-checkbox-hover-not-disabled": "oklch(65% 0.05 130)",
+    "--theme-checkbox-background-inherited": "oklch(85% 0.1 130)",
+    "--theme-table-border": "1px solid oklch(85% 0.1 130)",
+    "--theme-table-cell-background": "oklch(97% 0.02 130)",
+    "--theme-table-cell-empty-background": "oklch(95% 0.04 130)",
+    "--theme-table-cell-empty-foreground": "oklch(65% 0.05 130)",
+    "--theme-table-header-background": "oklch(95% 0.04 130)",
+    "--theme-table-selected-background": "oklch(75% 0.15 130)",
+    "--theme-table-active-background": "oklch(80% 0.1 130)",
+    "--theme-table-hover-background": "oklch(95% 0.04 130)",
+    "--theme-table-added-background": "oklch(95% 0.1 110)",
+    "--theme-table-changed-background": "oklch(95% 0.1 135)",
+    "--theme-table-deleted-background": "oklch(95% 0.1 25)",
+    "--theme-cell-active-border": "2px solid oklch(50% 0.25 130)",
+    "--theme-object-header-background": "oklch(95% 0.04 130)",
+    "--theme-modal-background": "oklch(97% 0.02 130)",
+    "--theme-modal-header-background": "oklch(85% 0.1 130)",
+    "--theme-modal-footer-background": "oklch(97% 0.02 130)",
+    "--theme-modal-border": "1px solid oklch(85% 0.1 130)",
+    "--theme-modal-overlay-background": "color-mix(in srgb, #124012 40%, transparent)",
+    "--theme-modal-shadow": "0 20px 25px -5px color-mix(in srgb, #124012 10%, transparent)",
+    "--theme-modal-close-hover-background": "oklch(70% 0.15 130)",
+    "--theme-formbutton-foreground": "white",
+    "--theme-formbutton-border": "1px solid oklch(40% 0.25 130)",
+    "--theme-formbutton-border-hover": "1px solid oklch(50% 0.3 130)",
+    "--theme-formbutton-border-active": "2px solid oklch(55% 0.35 130)",
+    "--theme-formbutton-background": "oklch(40% 0.25 130)",
+    "--theme-formbutton-background-disabled": "oklch(85% 0.1 130)",
+    "--theme-formbutton-border-disabled": "1px solid oklch(85% 0.1 130)",
+    "--theme-formbutton-foreground-disabled": "oklch(65% 0.05 130)",
+    "--theme-formbutton-background-hover": "oklch(35% 0.3 130)",
+    "--theme-formbutton-background-active": "oklch(35% 0.3 130)",
+    "--theme-outlinebutton-foreground": "oklch(10% 0.06 130)",
+    "--theme-outlinebutton-border": "1px solid oklch(40% 0.25 130)",
+    "--theme-outlinebutton-hover-foreground": "oklch(40% 0.25 130)",
+    "--theme-outlinebutton-hover-border": "2px solid oklch(50% 0.3 130)",
+    "--theme-tabs-control-background": "oklch(95% 0.04 130)",
+    "--theme-tabs-control-border": "1px solid oklch(90% 0.1 130)",
+    "--theme-tabs-control-selected-background": "oklch(98% 0.01 130)",
+    "--theme-tabs-control-selected-border": "2px solid oklch(50% 0.25 130)",
+    "--theme-inline-tabs-border": "1px solid oklch(90% 0.1 130)",
+    "--theme-inline-tabs-border-active": "2px solid oklch(50% 0.25 130)",
+    "--theme-toolstrip-background": "oklch(97% 0.02 130)",
+    "--theme-toolstrip-border": "1px solid oklch(90% 0.1 130)",
+    "--theme-toolstrip-button-foreground": "oklch(27% 0.07 130)",
+    "--theme-panel-border-subtle": "1px solid color-mix(in srgb, oklch(20% 0.06 130) 5%, transparent)",
+    "--theme-panel-type-label-color": "oklch(65% 0.05 130)",
+    "--theme-toolstrip-button-foreground-disabled": "oklch(65% 0.05 130)",
+    "--theme-toolstrip-button-foreground-icon": "oklch(40% 0.12 130)",
+    "--theme-toolstrip-button-background": "oklch(97% 0.02 130)",
+    "--theme-toolstrip-button-background-hover": "oklch(95% 0.04 130)",
+    "--theme-toolstrip-button-background-active": "oklch(90% 0.1 130)",
+    "--theme-toolstrip-button-border": "1px solid oklch(90% 0.1 130)",
+    "--theme-toolstrip-button-border-hover": "1px solid oklch(85% 0.1 130)",
+    "--theme-toolstrip-button-border-disabled": "1px solid oklch(90% 0.1 130)",
+    "--theme-toolstrip-button-split-separator-border": "1px solid oklch(85% 0.1 130)",
+    "--theme-designer-background": "oklch(97% 0.02 130)",
+    "--theme-designer-item-background": "oklch(95% 0.04 130)",
+    "--theme-designer-selection-marker": "oklch(35% 0.3 130)",
+    "--theme-designer-item-border": "1px solid oklch(90% 0.1 130)",
+    "--theme-designer-stroke-color": "oklch(65% 0.05 130)",
+    "--theme-designer-arrow-color": "oklch(27% 0.07 130)",
+    "--theme-designer-select-reactangle-foreground": "oklch(50% 0.25 130)",
+    "--theme-designer-header-background-1": "oklch(70% 0.15 130)",
+    "--theme-designer-header-background-2": "oklch(70% 0.18 180)",
+    "--theme-designer-header-background-3": "oklch(68% 0.15 100)",
+    "--theme-designer-header-background-grayed": "oklch(85% 0.1 130)",
+    "--theme-designer-close-background": "oklch(90% 0.1 130)",
+    "--theme-designer-close-background-hover": "oklch(85% 0.1 130)",
+    "--theme-designer-close-background-active": "oklch(70% 0.15 130)",
+    "--theme-designer-drag-column-background": "oklch(90% 0.2 110)",
+    "--theme-designer-select-column-background": "oklch(90% 0.1 130)",
+    "--theme-statusbar-background": "oklch(40% 0.25 130)",
+    "--theme-statusbar-foreground": "oklch(95% 0.04 130)",
+    "--theme-statusbar-background-hover": "oklch(35% 0.3 130)",
+    "--theme-statusbar-button-background": "oklch(85% 0.1 130)",
+    "--theme-statusbar-button-foreground": "oklch(27% 0.07 130)",
+    "--theme-statusbar-icon-error": "oklch(80% 0.1 25)",
+    "--theme-statusbar-icon-ok": "oklch(85% 0.2 130)",
+    "--theme-aichat-user-background": "oklch(93% 0.06 130)",
+    "--theme-aichat-assistant-background": "oklch(95% 0.04 130)",
+    "--theme-applog-details-background": "oklch(98% 0.01 130)",
+    "--theme-input-border": "1px solid oklch(85% 0.1 130)",
+    "--theme-input-border-hover": "1px solid oklch(70% 0.15 130)",
+    "--theme-input-border-hover-color": "oklch(70% 0.15 130)",
+    "--theme-input-border-focus": "1px solid oklch(50% 0.25 130)",
+    "--theme-input-border-focus-color": "oklch(50% 0.25 130)",
+    "--theme-input-border-disabled": "1px solid oklch(90% 0.1 130)",
+    "--theme-input-background": "white",
+    "--theme-input-foreground": "oklch(20% 0.06 130)",
+    "--theme-input-placeholder": "oklch(65% 0.05 130)",
+    "--theme-input-background-disabled": "oklch(95% 0.04 130)",
+    "--theme-input-foreground-disabled": "oklch(65% 0.05 130)",
+    "--theme-input-focus-ring": "0 0 0 3px color-mix(in srgb, oklch(50% 0.25 130) 10%, transparent)",
+    "--theme-input-multi-clear-background": "oklch(90% 0.1 130)",
+    "--theme-input-multi-clear-foreground": "oklch(40% 0.12 130)",
+    "--theme-input-multi-clear-hover": "oklch(85% 0.1 130)",
+    "--theme-input-shadow": "0 1px 2px 0 color-mix(in srgb, oklch(20% 0.06 130) 5%, transparent)",
+    "--theme-input-shadow-hover": "0 4px 6px -2px color-mix(in srgb, oklch(20% 0.06 130) 8%, transparent)",
+    "--theme-input-shadow-focus": "0 1px 2px 0 color-mix(in srgb, oklch(20% 0.06 130) 5%, transparent)",
+    "--theme-input-inplace-select-shadow": "0 1px 10px 1px oklch(40% 0.12 130)",
+    "--theme-color-selected-border": "2px solid oklch(27% 0.07 130)",
+    "--theme-new-object-button-background": "oklch(90% 0.1 130)",
+    "--theme-new-object-button-background-hover": "oklch(85% 0.1 130)",
+    "--theme-status-valid-background": "oklch(95% 0.1 110)",
+    "--theme-status-testing-background": "oklch(95% 0.1 135)",
+    "--theme-status-error-background": "oklch(95% 0.1 25)",
+    "--theme-status-unconfigured-background": "oklch(95% 0.04 130)",
+    "--theme-status-untested-background": "oklch(94% 0.1 65)",
+    "--theme-dropdown-icon-hover": "oklch(45% 0.3 130)",
+    "--theme-icon-picker-background": "oklch(90% 0.1 130)",
+    "--theme-icon-picker-border": "1px solid oklch(85% 0.1 130)",
+    "--theme-icon-picker-hover": "oklch(85% 0.1 130)",
+    "--theme-icon-picker-selected": "oklch(80% 0.15 130)",
+    "--theme-dbkey-background": "oklch(98% 0.01 130)",
+    "--theme-dbkey-border": "1px solid oklch(90% 0.1 130)",
+    "--theme-dbkey-icon-hover": "oklch(70% 0.15 130)",
+    "--theme-chip-background": "oklch(85% 0.1 130)",
+    "--theme-titlebar-background": "oklch(85% 0.1 130)",
+    "--theme-titlebar-button-hover": "oklch(70% 0.15 130)",
+    "--theme-card-background": "oklch(90% 0.1 130)",
+    "--theme-card-border": "1px solid oklch(85% 0.1 130)",
+    "--theme-content-background-hover": "oklch(95% 0.04 130)",
+    "--theme-admin-menu-item-hover": "oklch(95% 0.04 130)",
+    "--theme-admin-menu-item-active": "oklch(85% 0.1 130)",
+    "--theme-admin-menu-background": "oklch(90% 0.1 130)",
+    "--theme-admin-menu-border": "1px solid oklch(90% 0.1 130)",
+    "--theme-json-tree-string-color": "oklch(45% 0.3 110)",
+    "--theme-json-tree-symbol-color": "oklch(45% 0.3 110)",
+    "--theme-json-tree-boolean-color": "oklch(40% 0.25 130)",
+    "--theme-json-tree-function-color": "oklch(40% 0.25 130)",
+    "--theme-json-tree-number-color": "oklch(50% 0.3 130)",
+    "--theme-json-tree-label-color": "oklch(55% 0.3 140)",
+    "--theme-json-tree-arrow-color": "oklch(65% 0.05 130)",
+    "--theme-json-tree-null-color": "oklch(65% 0.05 130)",
+    "--theme-json-tree-undefined-color": "oklch(65% 0.05 130)",
+    "--theme-json-tree-date-color": "oklch(65% 0.05 130)",
+    "--theme-json-tree-deleted-background": "oklch(95% 0.1 25)",
+    "--theme-json-tree-modified-background": "oklch(95% 0.1 135)",
+    "--theme-json-tree-inserted-background": "oklch(95% 0.1 110)",
+    "--theme-icon-blue": "oklch(40% 0.25 130)",
+    "--theme-icon-green": "oklch(45% 0.2 140)",
+    "--theme-icon-red": "oklch(40% 0.3 25)",
+    "--theme-icon-gold": "oklch(50% 0.2 60)",
+    "--theme-icon-yellow": "oklch(50% 0.15 80)",
+    "--theme-icon-magenta": "oklch(45% 0.3 135)"
+  }
+}

e2e-tests/data/redis-db1.redis

@@ -0,0 +1,15 @@
+HSET "actor:1000" "first_name" "Sandra"
+HSET "actor:1000" "last_name" "Bullock"
+HSET "actor:1000" "date_of_birth" "1964"
+
+HSET "actor:1001" "first_name" "Jon"
+HSET "actor:1001" "last_name" "Hamm"
+HSET "actor:1001" "date_of_birth" "1971"
+
+HSET "actor:1002" "first_name" "Allison"
+HSET "actor:1002" "last_name" "Janney"
+HSET "actor:1002" "date_of_birth" "1959"
+
+HSET "actor:1003" "first_name" "Steve"
+HSET "actor:1003" "last_name" "Coogan"
+HSET "actor:1003" "date_of_birth" "1965"

e2e-tests/data/redis-db2.redis

@@ -0,0 +1,14 @@
+SET app:name "App"
+SET app:version "1.0.0"
+SET app:env "test"
+SET user:1:json "{\"id\":1,\"name\":\"Alice\",\"email\":\"alice@app.test\",\"roles\":[\"admin\",\"user\"],\"settings\":{\"theme\":\"dark\",\"language\":\"sk\"}}"
+SET user:2:json "{\"id\":2,\"name\":\"Bob\",\"email\":\"bob@app.test\",\"roles\":[\"user\"],\"settings\":{\"theme\":\"light\",\"language\":\"en\"}}"
+RPUSH queue:emails "welcome" "reset-password" "newsletter" "promotion" "weekly-digest"
+HSET user:alice name "Alice" email "alice@app.test" active "true" age "29" country "SK"
+HSET user:bob name "Bob" email "bob@app.test" active "false" age "34" country "CZ"
+SADD tags "app" "backend" "database" "redis" "test" "production"
+ZADD leaderboard 100 "alice" 250 "bob" 180 "carol" 90 "dave" 300 "eve"
+XADD events * type "login" userId "1" ip "127.0.0.1" device "web"
+XADD events * type "update-profile" userId "1" field "email" old "alice@app.test" new "alice@new.app"
+XADD events * type "login" userId "2" ip "10.0.0.5" device "mobile"
+XADD events * type "logout" userId "1" reason "manual"

e2e-tests/env/browse-data/.env

@@ -1,4 +1,4 @@
-CONNECTIONS=mysql,postgres,mongo,redis
+CONNECTIONS=mysql,postgres,mongo
 
 LABEL_mysql=MySql-connection
 SERVER_mysql=localhost
@@ -22,8 +22,3 @@ USER_mongo=root
 PASSWORD_mongo=Pwd2020Db
 PORT_mongo=16010
 ENGINE_mongo=mongo@dbgate-plugin-mongo
-
-LABEL_redis=Redis-connection
-SERVER_redis=localhost
-ENGINE_redis=redis@dbgate-plugin-redis
-PORT_redis=16011

e2e-tests/env/redis/.env

@@ -0,0 +1,6 @@
+CONNECTIONS=redis
+
+LABEL_redis=Redis-connection
+SERVER_redis=localhost
+ENGINE_redis=redis@dbgate-plugin-redis
+PORT_redis=16011

e2e-tests/init/browse-data.js

@@ -125,46 +125,6 @@ async function initMongoDatabase(dbname, inputDirectory) {
   // });
 }
 
-async function initRedisDatabase(inputDirectory) {
-  await dbgateApi.executeQuery({
-    connection: {
-      server: process.env.SERVER_redis,
-      user: process.env.USER_redis,
-      password: process.env.PASSWORD_redis,
-      port: process.env.PORT_redis,
-      engine: 'redis@dbgate-plugin-redis',
-    },
-    sql: 'FLUSHALL',
-  });
-
-  for (const file of fs.readdirSync(inputDirectory)) {
-    await dbgateApi.executeQuery({
-      connection: {
-        server: process.env.SERVER_redis,
-        user: process.env.USER_redis,
-        password: process.env.PASSWORD_redis,
-        port: process.env.PORT_redis,
-        engine: 'redis@dbgate-plugin-redis',
-        database: 0,
-      },
-      sqlFile: path.join(inputDirectory, file),
-      // logScriptItems: true,
-    });
-  }
-
-  // await dbgateApi.importDatabase({
-  //   connection: {
-  //     server: process.env.SERVER_postgres,
-  //     user: process.env.USER_postgres,
-  //     password: process.env.PASSWORD_postgres,
-  //     port: process.env.PORT_postgres,
-  //     database: dbname,
-  //     engine: 'postgres@dbgate-plugin-postgres',
-  //   },
-  //   inputFile,
-  // });
-}
-
 const baseDir = path.join(os.homedir(), '.dbgate');
 
 async function copyFolder(source, target) {
@@ -188,8 +148,6 @@ async function run() {
   await initMongoDatabase('MgChinook', path.resolve(path.join(__dirname, '../data/chinook-jsonl')));
   await initMongoDatabase('MgRivers', path.resolve(path.join(__dirname, '../data/rivers-jsonl')));
 
-  await initRedisDatabase(path.resolve(path.join(__dirname, '../data/redis')));
-
   await copyFolder(
     path.resolve(path.join(__dirname, '../data/chinook-jsonl')),
     path.join(baseDir, 'archive-e2etests', 'default')

e2e-tests/init/charts.js

@@ -90,6 +90,11 @@ async function run() {
     path.join(baseDir, 'files-e2etests', 'sql')
   );
 
+  await copyFolder(
+    path.resolve(path.join(__dirname, '../data/files/themes')),
+    path.join(baseDir, 'files-e2etests', 'themes')
+  );
+
   await initMySqlDatabase('MyChinook', path.resolve(path.join(__dirname, '../data/chinook-mysql.sql')));
 }
 

e2e-tests/init/redis.js

@@ -0,0 +1,55 @@
+const path = require('path');
+const fs = require('fs');
+
+const dbgateApi = require('dbgate-api');
+dbgateApi.initializeApiEnvironment();
+const dbgatePluginRedis = require('dbgate-plugin-redis');
+dbgateApi.registerPlugins(dbgatePluginRedis);
+
+async function initRedisDatabase() {
+  await dbgateApi.executeQuery({
+    connection: {
+      server: process.env.SERVER_redis,
+      user: process.env.USER_redis,
+      password: process.env.PASSWORD_redis,
+      port: process.env.PORT_redis,
+      engine: 'redis@dbgate-plugin-redis',
+    },
+    sql: 'FLUSHALL',
+  });
+
+  const files = [
+    {
+      file: path.resolve(__dirname, '../data/redis-db1.redis'),
+      database: 0,
+    },
+    {
+      file: path.resolve(__dirname, '../data/redis-db2.redis'),
+      database: 1,
+    },
+  ];
+
+  for (const { file, database } of files) {
+    await dbgateApi.executeQuery({
+      connection: {
+        server: process.env.SERVER_redis,
+        user: process.env.USER_redis,
+        password: process.env.PASSWORD_redis,
+        port: process.env.PORT_redis,
+        engine: 'redis@dbgate-plugin-redis',
+        database,
+      },
+      sqlFile: file,
+    });
+  }
+}
+
+async function run() {
+  await initRedisDatabase();
+}
+
+dbgateApi.runScript(run);
+
+module.exports = {
+  initRedisDatabase,
+};

e2e-tests/package.json

@@ -10,11 +10,11 @@
     "cypress-real-events": "^1.13.0",
     "env-cmd": "^10.1.0",
     "kill-port": "^2.0.1",
+    "mocha-reporter-gha": "^1.1.1",
     "start-server-and-test": "^2.0.8"
   },
   "scripts": {
     "cy:open": "cypress open --config experimentalInteractiveRunEvents=true",
-
     "cy:run:add-connection": "cypress run --spec cypress/e2e/add-connection.cy.js",
     "cy:run:portal": "cypress run --spec cypress/e2e/portal.cy.js",
     "cy:run:oauth": "cypress run --spec cypress/e2e/oauth.cy.js",
@@ -23,7 +23,7 @@
     "cy:run:multi-sql": "cypress run --spec cypress/e2e/multi-sql.cy.js",
     "cy:run:cloud": "cypress run --spec cypress/e2e/cloud.cy.js",
     "cy:run:charts": "cypress run --spec cypress/e2e/charts.cy.js",
-
+    "cy:run:redis": "cypress run --spec cypress/e2e/redis.cy.js",
     "start:add-connection": "node clearTestingData && cd .. && node packer/build/bundle.js --listen-api --run-e2e-tests",
     "start:portal": "node clearTestingData && cd .. && env-cmd -f e2e-tests/env/portal/.env node e2e-tests/init/portal.js && env-cmd -f e2e-tests/env/portal/.env node packer/build/bundle.js --listen-api --run-e2e-tests",
     "start:oauth": "node clearTestingData && cd .. && env-cmd -f e2e-tests/env/oauth/.env node packer/build/bundle.js --listen-api --run-e2e-tests",
@@ -32,7 +32,7 @@
     "start:multi-sql": "node clearTestingData && cd .. && env-cmd -f e2e-tests/env/multi-sql/.env node e2e-tests/init/multi-sql.js && env-cmd -f e2e-tests/env/multi-sql/.env node packer/build/bundle.js --listen-api --run-e2e-tests",
     "start:cloud": "node clearTestingData && cd .. && env-cmd -f e2e-tests/env/cloud/.env node packer/build/bundle.js --listen-api --run-e2e-tests",
     "start:charts": "node clearTestingData && cd .. && env-cmd -f e2e-tests/env/charts/.env node e2e-tests/init/charts.js && env-cmd -f e2e-tests/env/charts/.env node packer/build/bundle.js --listen-api --run-e2e-tests",
-
+    "start:redis": "node clearTestingData && cd .. && env-cmd -f e2e-tests/env/redis/.env node e2e-tests/init/redis.js && env-cmd -f e2e-tests/env/redis/.env node packer/build/bundle.js --listen-api --run-e2e-tests",
     "test:add-connection": "start-server-and-test start:add-connection http://localhost:3000 cy:run:add-connection",
     "test:portal": "start-server-and-test start:portal http://localhost:3000 cy:run:portal",
     "test:oauth": "start-server-and-test start:oauth http://localhost:3000 cy:run:oauth",
@@ -41,8 +41,8 @@
     "test:multi-sql": "start-server-and-test start:multi-sql http://localhost:3000 cy:run:multi-sql",
     "test:cloud": "start-server-and-test start:cloud http://localhost:3000 cy:run:cloud",
     "test:charts": "start-server-and-test start:charts http://localhost:3000 cy:run:charts",
-
-    "test": "yarn test:add-connection && yarn test:portal && yarn test:oauth && yarn test:browse-data && yarn test:team && yarn test:multi-sql && yarn test:cloud && yarn test:charts",
+    "test:redis": "start-server-and-test start:redis http://localhost:3000 cy:run:redis",
+    "test": "yarn test:add-connection && yarn test:portal && yarn test:oauth && yarn test:browse-data && yarn test:team && yarn test:multi-sql && yarn test:cloud && yarn test:charts && yarn test:redis",
     "test:ci": "yarn test"
   },
   "dependencies": {}

e2e-tests/yarn.lock

@@ -2,6 +2,34 @@
 # yarn lockfile v1
 
 
+"@actions/core@^1.10.1":
+  version "1.11.1"
+  resolved "https://registry.yarnpkg.com/@actions/core/-/core-1.11.1.tgz#ae683aac5112438021588030efb53b1adb86f172"
+  integrity sha512-hXJCSrkwfA46Vd9Z3q4cpEpHB1rL5NG04+/rbqW9d3+CSvtB1tYe8UTpAlixa1vj0m/ULglfEK2UKxMGxCxv5A==
+  dependencies:
+    "@actions/exec" "^1.1.1"
+    "@actions/http-client" "^2.0.1"
+
+"@actions/exec@^1.1.1":
+  version "1.1.1"
+  resolved "https://registry.yarnpkg.com/@actions/exec/-/exec-1.1.1.tgz#2e43f28c54022537172819a7cf886c844221a611"
+  integrity sha512-+sCcHHbVdk93a0XT19ECtO/gIXoxvdsgQLzb2fE2/5sIZmWQuluYyjPQtrtTHdU1YzTZ7bAPN4sITq2xi1679w==
+  dependencies:
+    "@actions/io" "^1.0.1"
+
+"@actions/http-client@^2.0.1":
+  version "2.2.3"
+  resolved "https://registry.yarnpkg.com/@actions/http-client/-/http-client-2.2.3.tgz#31fc0b25c0e665754ed39a9f19a8611fc6dab674"
+  integrity sha512-mx8hyJi/hjFvbPokCg4uRd4ZX78t+YyRPtnKWwIl+RzNaVuFpQHfmlGVfsKEJN8LwTCvL+DfVgAM04XaHkm6bA==
+  dependencies:
+    tunnel "^0.0.6"
+    undici "^5.25.4"
+
+"@actions/io@^1.0.1":
+  version "1.1.3"
+  resolved "https://registry.yarnpkg.com/@actions/io/-/io-1.1.3.tgz#4cdb6254da7962b07473ff5c335f3da485d94d71"
+  integrity sha512-wi9JjgKLYS7U/z8PPbco+PvTb/nRWjeoFlJ1Qer83k/3C5PHQi28hiVdeE2kHXmIL99mQFawx8qt/JPjZilJ8Q==
+
 "@colors/colors@1.5.0":
   version "1.5.0"
   resolved "https://registry.yarnpkg.com/@colors/colors/-/colors-1.5.0.tgz#bb504579c1cae923e6576a4f5da43d25f97bdbd9"
@@ -39,6 +67,11 @@
     debug "^3.1.0"
     lodash.once "^4.1.1"
 
+"@fastify/busboy@^2.0.0":
+  version "2.1.1"
+  resolved "https://registry.yarnpkg.com/@fastify/busboy/-/busboy-2.1.1.tgz#b9da6a878a371829a0502c9b6c1c143ef6663f4d"
+  integrity sha512-vBZP4NlzfOlerQTnba4aqZoMhE/a9HY7HRqoOPaETQcSQuWEIyZMHGfVu6w9wGtGK5fED5qRs2DteVCjOH60sA==
+
 "@hapi/hoek@^9.0.0", "@hapi/hoek@^9.3.0":
   version "9.3.0"
   resolved "https://registry.yarnpkg.com/@hapi/hoek/-/hoek-9.3.0.tgz#8368869dcb735be2e7f5cb7647de78e167a251fb"
@@ -947,6 +980,13 @@ minimist@^1.2.8:
   resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.8.tgz#c1a464e7693302e082a075cee0c057741ac4772c"
   integrity sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==
 
+mocha-reporter-gha@^1.1.1:
+  version "1.1.1"
+  resolved "https://registry.yarnpkg.com/mocha-reporter-gha/-/mocha-reporter-gha-1.1.1.tgz#e1248abd0769f55b57b36ccd7db2b0b6573d5adf"
+  integrity sha512-CFbcgM56V4yWlbF91XuwrE6a5X/IqjVXTPefO7m8cY8Es8G1UhJ2KKOrk16AcSemRzVWXp2Fdy3bWJ7j45snWw==
+  dependencies:
+    "@actions/core" "^1.10.1"
+
 ms@^2.1.1, ms@^2.1.3:
   version "2.1.3"
   resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.3.tgz#574c8138ce1d2b5861f0b44579dbadd60c6615b2"
@@ -1292,6 +1332,11 @@ tunnel-agent@^0.6.0:
   dependencies:
     safe-buffer "^5.0.1"
 
+tunnel@^0.0.6:
+  version "0.0.6"
+  resolved "https://registry.yarnpkg.com/tunnel/-/tunnel-0.0.6.tgz#72f1314b34a5b192db012324df2cc587ca47f92c"
+  integrity sha512-1h/Lnq9yajKY2PEbBadPXj3VxsDDu844OnaAo52UVmIzIvwwtBPIuNvkjuzBlTWpfJyUbG3ez0KSBibQkj4ojg==
+
 tweetnacl@^0.14.3, tweetnacl@~0.14.0:
   version "0.14.5"
   resolved "https://registry.yarnpkg.com/tweetnacl/-/tweetnacl-0.14.5.tgz#5ae68177f192d4456269d108afa93ff8743f4f64"
@@ -1307,6 +1352,13 @@ undici-types@~6.20.0:
   resolved "https://registry.yarnpkg.com/undici-types/-/undici-types-6.20.0.tgz#8171bf22c1f588d1554d55bf204bc624af388433"
   integrity sha512-Ny6QZ2Nju20vw1SRHe3d9jVu6gJ+4e3+MMpqu7pqE5HT6WsTSlce++GQmK5UXS8mzV8DSYHrQH+Xrf2jVcuKNg==
 
+undici@^5.25.4:
+  version "5.29.0"
+  resolved "https://registry.yarnpkg.com/undici/-/undici-5.29.0.tgz#419595449ae3f2cdcba3580a2e8903399bd1f5a3"
+  integrity sha512-raqeBD6NQK4SkWhQzeYKd1KmIG6dllBOTt55Rmkt4HtI9mwdWtJljnrXjAFUBLTSN67HWrOIZ3EPF4kjUw80Bg==
+  dependencies:
+    "@fastify/busboy" "^2.0.0"
+
 universalify@^2.0.0:
   version "2.0.1"
   resolved "https://registry.yarnpkg.com/universalify/-/universalify-2.0.1.tgz#168efc2180964e6386d061e094df61afe239b18d"

integration-tests/__tests__/alter-table.spec.js

@@ -26,13 +26,15 @@ function pickImportantTableInfo(engine, table) {
       .map(props =>
         _.omitBy(props, (v, k) => k == 'defaultValue' && v == 'NULL' && engine.setNullDefaultInsteadOfDrop)
       ),
-    // foreignKeys: table.foreignKeys
-    //   .sort((a, b) => a.refTableName.localeCompare(b.refTableName))
-    //   .map(fk => ({
-    //     constraintType: fk.constraintType,
-    //     refTableName: fk.refTableName,
-    //     columns: fk.columns.map(col => ({ columnName: col.columnName, refColumnName: col.refColumnName })),
-    //   })),
+
+      // TODO:
+    foreignKeys: table.foreignKeys
+      .sort((a, b) => a.refTableName.localeCompare(b.refTableName))
+      .map(fk => ({
+        constraintType: fk.constraintType,
+        refTableName: fk.refTableName,
+        columns: fk.columns.map(col => ({ columnName: col.columnName, refColumnName: col.refColumnName })),
+      })),
   };
 }
 
@@ -103,6 +105,7 @@ async function testTableDiff(engine, conn, driver, mangle, changedTable = 't1')
 
   await driver.script(conn, sql);
 
+  // TODO:
   // if (!engine.skipIncrementalAnalysis) {
   //   const structure2RealIncremental = await driver.analyseIncremental(conn, structure1Source);
   //   checkTableStructure(engine, tget(structure2RealIncremental), tget(structure2));
@@ -116,6 +119,7 @@ async function testTableDiff(engine, conn, driver, mangle, changedTable = 't1')
 
 const TESTED_COLUMNS = ['col_pk', 'col_std', 'col_def', 'col_fk', 'col_ref', 'col_idx', 'col_uq'];
 // const TESTED_COLUMNS = ['col_pk'];
+// const TESTED_COLUMNS = ['col_fk'];
 // const TESTED_COLUMNS = ['col_idx'];
 // const TESTED_COLUMNS = ['col_def'];
 // const TESTED_COLUMNS = ['col_std'];
@@ -179,11 +183,25 @@ describe('Alter table', () => {
   )(
     'Drop column - %s - %s',
     testWrapper(async (conn, driver, column, engine) => {
-      await testTableDiff(engine, conn, driver, tbl => (tbl.columns = tbl.columns.filter(x => x.columnName != column)));
+      await testTableDiff(engine, conn, driver, 
+        tbl => {
+          tbl.columns = tbl.columns.filter(x => x.columnName != column);
+          tbl.foreignKeys = tbl.foreignKeys
+            .map(fk => ({
+              ...fk, 
+              columns: fk.columns.filter(col => col.columnName != column)
+            }))
+            .filter(fk => fk.columns.length > 0);
+        }
+      );
     })
   );
 
-  test.each(createEnginesColumnsSource(engines.filter(x => !x.skipNullable && !x.skipChangeNullability)))(
+  test.each(
+    createEnginesColumnsSource(engines.filter(x => !x.skipNullability && !x.skipChangeNullability)).filter(
+      ([_label, col]) => !col.endsWith('_pk')
+    )
+  )(
     'Change nullability - %s - %s',
     testWrapper(async (conn, driver, column, engine) => {
       await testTableDiff(
@@ -202,7 +220,11 @@ describe('Alter table', () => {
         engine,
         conn,
         driver,
-        tbl => (tbl.columns = tbl.columns.map(x => (x.columnName == column ? { ...x, columnName: 'col_renamed' } : x)))
+        tbl => {
+          tbl.columns = tbl.columns.map(x => (x.columnName == column ? { ...x, columnName: 'col_renamed' } : x));
+          tbl.foreignKeys = tbl.foreignKeys.map(fk => ({...fk, columns: fk.columns.map(col => col.columnName == column ? { ...col, columnName: 'col_renamed' } : col)
+          }));
+        }
       );
     })
   );

integration-tests/__tests__/data-replicator.spec.js

@@ -303,4 +303,52 @@ describe('Data replicator', () => {
     }),
     15 * 1000
   );
+
+  test.each(engines.filter(x => !x.skipDataReplicator).map(engine => [engine.label, engine]))(
+    'Skip columns for update - %s',
+    testWrapper(async (conn, driver, engine) => {
+      runCommandOnDriver(conn, driver, dmp =>
+        dmp.createTable({
+          pureName: 't1',
+          columns: [
+            { columnName: 'id', dataType: 'int', autoIncrement: true, notNull: true },
+            { columnName: 'key', dataType: 'varchar(50)', notNull: true },
+            { columnName: 'val', dataType: 'varchar(50)' },
+          ],
+          primaryKey: {
+            columns: [{ columnName: 'id' }],
+          },
+        })
+      );
+
+      const getcfg = (v1 = 'v1') => ({
+        systemConnection: conn,
+        driver,
+        items: [
+          {
+            name: 't1',
+            matchColumns: ['key'],
+            skipUpdateColumns: ['val'],
+            findExisting: true,
+            updateExisting: true,
+            createNew: true,
+            jsonArray: [
+              { key: '1', val: v1 },
+              { key: '2', val: 'v2' },
+              { key: '3', val: 'v3' },
+            ],
+          },
+        ],
+      });
+
+      await dataReplicator(getcfg('v1'));
+
+      const res1 = await runQueryOnDriver(conn, driver, dmp => dmp.put(`select ~val from ~t1 where ~key='1'`));
+      expect(res1.rows[0].val).toEqual('v1');
+
+      await dataReplicator(getcfg('v2'));
+      const res2 = await runQueryOnDriver(conn, driver, dmp => dmp.put(`select ~val from ~t1 where ~key='1'`));
+      expect(res2.rows[0].val).toEqual('v1');
+    })
+  );
 });

integration-tests/__tests__/schema-tests.spec.js

@@ -28,12 +28,12 @@ describe('Schema tests', () => {
       const count = schemas1.length;
       expect(structure1.tables.length).toEqual(2);
       await runCommandOnDriver(conn, driver, dmp => dmp.createSchema('myschema'));
+      const structure2 = await driver.analyseIncremental(conn, structure1);
+      const schemas2 = await driver.listSchemas(conn);
+      expect(schemas2.find(x => x.schemaName == 'myschema')).toBeTruthy();
+      expect(schemas2.length).toEqual(count + 1);
+      expect(schemas2.find(x => x.isDefault).schemaName).toEqual(engine.defaultSchemaName);
       if (!engine.skipIncrementalAnalysis) {
-        const structure2 = await driver.analyseIncremental(conn, structure1);
-        const schemas2 = await driver.listSchemas(conn);
-        expect(schemas2.find(x => x.schemaName == 'myschema')).toBeTruthy();
-        expect(schemas2.length).toEqual(count + 1);
-        expect(schemas2.find(x => x.isDefault).schemaName).toEqual(engine.defaultSchemaName);
         expect(structure2).toBeNull();
       }
     })
@@ -50,10 +50,10 @@ describe('Schema tests', () => {
       expect(schemas1.find(x => x.schemaName == 'myschema')).toBeTruthy();
       expect(structure1.tables.length).toEqual(2);
       await runCommandOnDriver(conn, driver, dmp => dmp.dropSchema('myschema'));
+      const structure2 = await driver.analyseIncremental(conn, structure1);
+      const schemas2 = await driver.listSchemas(conn);
+      expect(schemas2.find(x => x.schemaName == 'myschema')).toBeFalsy();
       if (!engine.skipIncrementalAnalysis) {
-        const structure2 = await driver.analyseIncremental(conn, structure1);
-        const schemas2 = await driver.listSchemas(conn);
-        expect(schemas2.find(x => x.schemaName == 'myschema')).toBeFalsy();
         expect(structure2).toBeNull();
       }
     })

integration-tests/__tests__/table-analyse.spec.js

@@ -94,7 +94,7 @@ describe('Table analyse', () => {
     })
   );
 
-  test.each(engines.filter(x => !x.skipIncrementalAnalysis).map(engine => [engine.label, engine]))(
+  test.each(engines.map(engine => [engine.label, engine]))(
     'Table add - incremental analysis - %s',
     testWrapper(async (conn, driver, engine) => {
       await runCommandOnDriver(conn, driver, dmp => dmp.put(t2Sql(engine)));
@@ -112,7 +112,7 @@ describe('Table analyse', () => {
     })
   );
 
-  test.each(engines.filter(x => !x.skipIncrementalAnalysis).map(engine => [engine.label, engine]))(
+  test.each(engines.map(engine => [engine.label, engine]))(
     'Table remove - incremental analysis - %s',
     testWrapper(async (conn, driver, engine) => {
       await runCommandOnDriver(conn, driver, dmp => dmp.put(t1Sql(engine)));
@@ -130,7 +130,7 @@ describe('Table analyse', () => {
     })
   );
 
-  test.each(engines.filter(x => !x.skipIncrementalAnalysis).map(engine => [engine.label, engine]))(
+  test.each(engines.map(engine => [engine.label, engine]))(
     'Table change - incremental analysis - %s',
     testWrapper(async (conn, driver, engine) => {
       await runCommandOnDriver(conn, driver, dmp => dmp.put(t1Sql(engine)));

integration-tests/docker-compose.yaml

@@ -44,7 +44,7 @@ services:
   #     - 15942:9042
   #
   # clickhouse:
-  #   image: bitnami/clickhouse:24.8.4
+  #   image: bitnamilegacy/clickhouse:24.8.4
   #   restart: always
   #   ports:
   #     - 15005:8123

integration-tests/package.json

@@ -1,6 +1,6 @@
 {
   "name": "dbgate-integration-tests",
-  "version": "6.0.0-alpha.1",
+  "version": "7.0.0-alpha.1",
   "homepage": "https://dbgate.org/",
   "repository": {
     "type": "git",

integration-tests/setupTests.js

@@ -1,6 +1,7 @@
 global.DBGATE_PACKAGES = {
   'dbgate-tools': require('dbgate-tools'),
   'dbgate-sqltree': require('dbgate-sqltree'),
+  'dbgate-datalib': require('dbgate-datalib'),
 };
 
 const { prettyFactory } = require('pino-pretty');

integration-tests/tools.js

@@ -22,7 +22,9 @@ async function connect(engine, database) {
   if (engine.generateDbFile) {
     const conn = await driver.connect({
       ...connection,
-      databaseFile: (engine.databaseFileLocationOnServer ?? 'dbtemp/') + database,
+      databaseFile:
+        (engine.databaseFileLocationOnServer ?? (process.env.CITEST ? 'dbtemp/' : 'integration-tests/dbtemp/')) +
+        database,
     });
     return conn;
   } else {

integration-tests/wait.js

@@ -3,6 +3,7 @@ const engines = require('./engines');
 global.DBGATE_PACKAGES = {
   'dbgate-tools': require('dbgate-tools'),
   'dbgate-sqltree': require('dbgate-sqltree'),
+  'dbgate-datalib': require('dbgate-datalib'),
 };
 
 async function connectEngine(engine) {

package.json

@@ -1,6 +1,6 @@
 {
   "private": true,
-  "version": "6.7.2",
+  "version": "7.0.5-beta.1",
   "name": "dbgate-all",
   "workspaces": [
     "packages/*",
@@ -22,6 +22,7 @@
     "start:api:auth": "yarn workspace dbgate-api start:auth | pino-pretty",
     "start:api:dblogin": "yarn workspace dbgate-api start:dblogin | pino-pretty",
     "start:api:storage": "yarn workspace dbgate-api start:storage | pino-pretty",
+    "start:api:sfill": "yarn workspace dbgate-api start:sfill | pino-pretty",
     "start:api:storage:built": "yarn workspace dbgate-api start:storage:built | pino-pretty",
     "start:api:azure": "yarn workspace dbgate-api start:azure | pino-pretty",
     "start:api:e2e:team": "yarn workspace dbgate-api start:e2e:team | pino-pretty",

packages/api/env/sfill/.env

@@ -0,0 +1,54 @@
+DEVMODE=1
+DEVWEB=1 
+
+# STORAGE_SERVER=localhost
+# STORAGE_USER=root
+# STORAGE_PASSWORD=Pwd2020Db
+# STORAGE_PORT=3306
+# STORAGE_DATABASE=dbgate-filled
+# STORAGE_ENGINE=mysql@dbgate-plugin-mysql
+
+STORAGE_SERVER=localhost
+STORAGE_USER=postgres
+STORAGE_PASSWORD=Pwd2020Db
+STORAGE_PORT=5432
+STORAGE_DATABASE=dbgate_sfill
+STORAGE_ENGINE=postgres@dbgate-plugin-postgres
+
+
+CONNECTIONS=mysql,postgres,mongo,redis
+
+LABEL_mysql=MySql
+SERVER_mysql=dbgatedckstage1.sprinx.cz
+USER_mysql=root
+PASSWORD_mysql=Pwd2020Db
+PORT_mysql=3306
+ENGINE_mysql=mysql@dbgate-plugin-mysql
+
+LABEL_postgres=Postgres
+SERVER_postgres=dbgatedckstage1.sprinx.cz
+USER_postgres=postgres
+PASSWORD_postgres=Pwd2020Db
+PORT_postgres=5432
+ENGINE_postgres=postgres@dbgate-plugin-postgres
+
+LABEL_mongo=Mongo
+SERVER_mongo=dbgatedckstage1.sprinx.cz
+USER_mongo=root
+PASSWORD_mongo=Pwd2020Db
+PORT_mongo=27017
+ENGINE_mongo=mongo@dbgate-plugin-mongo
+
+LABEL_redis=Redis
+SERVER_redis=dbgatedckstage1.sprinx.cz
+ENGINE_redis=redis@dbgate-plugin-redis
+PORT_redis=6379
+
+ROLE_test1_CONNECTIONS=mysql
+ROLE_test1_PERMISSIONS=widgets/*
+ROLE_test1_DATABASES_db1_CONNECTION=mysql
+ROLE_test1_DATABASES_db1_PERMISSION=run_script
+ROLE_test1_DATABASES_db1_DATABASES=db1
+ROLE_test1_DATABASES_db2_CONNECTION=redis
+ROLE_test1_DATABASES_db2_PERMISSION=run_script
+ROLE_test1_DATABASES_db2_DATABASES=db2 

packages/api/package.json

@@ -1,7 +1,7 @@
 {
   "name": "dbgate-api",
   "main": "src/index.js",
-  "version": "6.0.0-alpha.1",
+  "version": "7.0.0-alpha.1",
   "homepage": "https://dbgate.org/",
   "repository": {
     "type": "git",
@@ -24,16 +24,16 @@
     "activedirectory2": "^2.1.0",
     "archiver": "^7.0.1",
     "async-lock": "^1.2.6",
-    "axios": "^0.21.1",
+    "axios": "^1.13.2",
     "body-parser": "^1.19.0",
     "byline": "^5.0.0",
     "compare-versions": "^3.6.0",
     "cors": "^2.8.5",
     "cross-env": "^6.0.3",
-    "dbgate-datalib": "^6.0.0-alpha.1",
+    "dbgate-datalib": "^7.0.0-alpha.1",
     "dbgate-query-splitter": "^4.11.9",
-    "dbgate-sqltree": "^6.0.0-alpha.1",
-    "dbgate-tools": "^6.0.0-alpha.1",
+    "dbgate-sqltree": "^7.0.0-alpha.1",
+    "dbgate-tools": "^7.0.0-alpha.1",
     "debug": "^4.3.4",
     "diff": "^5.0.0",
     "diff2html": "^3.4.13",
@@ -75,6 +75,7 @@
     "start:dblogin": "env-cmd -f env/dblogin/.env node src/index.js --listen-api",
     "start:filedb": "env-cmd node src/index.js /home/jena/test/chinook/Chinook.db --listen-api",
     "start:storage": "env-cmd -f env/storage/.env node src/index.js --listen-api",
+    "start:sfill": "env-cmd -f env/sfill/.env node src/index.js --listen-api",
     "start:storage:built": "env-cmd -f env/storage/.env cross-env DEVMODE= BUILTWEBMODE=1 node dist/bundle.js --listen-api",
     "start:singleconn": "env-cmd node src/index.js --server localhost --user root --port 3307 --engine mysql@dbgate-plugin-mysql --password test --listen-api",
     "start:azure": "env-cmd -f env/azure/.env node src/index.js --listen-api",
@@ -86,7 +87,7 @@
   "devDependencies": {
     "@types/fs-extra": "^9.0.11",
     "@types/lodash": "^4.14.149",
-    "dbgate-types": "^6.0.0-alpha.1",
+    "dbgate-types": "^7.0.0-alpha.1",
     "env-cmd": "^10.1.0",
     "jsdoc-to-markdown": "^9.0.5",
     "node-loader": "^1.0.2",

packages/api/src/controllers/auth.js

@@ -55,6 +55,8 @@ function authMiddleware(req, res, next) {
     '/stream',
     '/storage/get-connections-for-login-page',
     '/storage/set-admin-password',
+    '/storage/request-password-reset',
+    '/storage/reset-password',
     '/auth/get-providers',
     '/connections/dblogin-web',
     '/connections/dblogin-app',

packages/api/src/controllers/connections.js

@@ -23,10 +23,13 @@ const pipeForkLogs = require('../utility/pipeForkLogs');
 const requireEngineDriver = require('../utility/requireEngineDriver');
 const { getAuthProviderById } = require('../auth/authProvider');
 const { startTokenChecking } = require('../utility/authProxy');
+const { extractConnectionsFromEnv } = require('../utility/envtools');
+const { MissingCredentialsError } = require('../utility/exceptions');
 
 const logger = getLogger('connections');
 
 let volatileConnections = {};
+let pendingTestSubprocesses = {}; // Map of conid -> subprocess for MS Entra auth flows
 
 function getNamedArgs() {
   const res = {};
@@ -61,55 +64,7 @@ function getDatabaseFileLabel(databaseFile) {
 
 function getPortalCollections() {
   if (process.env.CONNECTIONS) {
-    const connections = _.compact(process.env.CONNECTIONS.split(',')).map(id => ({
-      _id: id,
-      engine: process.env[`ENGINE_${id}`],
-      server: process.env[`SERVER_${id}`],
-      user: process.env[`USER_${id}`],
-      password: process.env[`PASSWORD_${id}`],
-      passwordMode: process.env[`PASSWORD_MODE_${id}`],
-      port: process.env[`PORT_${id}`],
-      databaseUrl: process.env[`URL_${id}`],
-      useDatabaseUrl: !!process.env[`URL_${id}`],
-      databaseFile: process.env[`FILE_${id}`]?.replace(
-        '%%E2E_TEST_DATA_DIRECTORY%%',
-        path.join(path.dirname(path.dirname(__dirname)), 'e2e-tests', 'tmpdata')
-      ),
-      socketPath: process.env[`SOCKET_PATH_${id}`],
-      serviceName: process.env[`SERVICE_NAME_${id}`],
-      authType: process.env[`AUTH_TYPE_${id}`] || (process.env[`SOCKET_PATH_${id}`] ? 'socket' : undefined),
-      defaultDatabase:
-        process.env[`DATABASE_${id}`] ||
-        (process.env[`FILE_${id}`] ? getDatabaseFileLabel(process.env[`FILE_${id}`]) : null),
-      singleDatabase: !!process.env[`DATABASE_${id}`] || !!process.env[`FILE_${id}`],
-      displayName: process.env[`LABEL_${id}`],
-      isReadOnly: process.env[`READONLY_${id}`],
-      databases: process.env[`DBCONFIG_${id}`] ? safeJsonParse(process.env[`DBCONFIG_${id}`]) : null,
-      allowedDatabases: process.env[`ALLOWED_DATABASES_${id}`]?.replace(/\|/g, '\n'),
-      allowedDatabasesRegex: process.env[`ALLOWED_DATABASES_REGEX_${id}`],
-      parent: process.env[`PARENT_${id}`] || undefined,
-      useSeparateSchemas: !!process.env[`USE_SEPARATE_SCHEMAS_${id}`],
-      localDataCenter: process.env[`LOCAL_DATA_CENTER_${id}`],
-
-      // SSH tunnel
-      useSshTunnel: process.env[`USE_SSH_${id}`],
-      sshHost: process.env[`SSH_HOST_${id}`],
-      sshPort: process.env[`SSH_PORT_${id}`],
-      sshMode: process.env[`SSH_MODE_${id}`],
-      sshLogin: process.env[`SSH_LOGIN_${id}`],
-      sshPassword: process.env[`SSH_PASSWORD_${id}`],
-      sshKeyfile: process.env[`SSH_KEY_FILE_${id}`],
-      sshKeyfilePassword: process.env[`SSH_KEY_FILE_PASSWORD_${id}`],
-
-      // SSL
-      useSsl: process.env[`USE_SSL_${id}`],
-      sslCaFile: process.env[`SSL_CA_FILE_${id}`],
-      sslCertFile: process.env[`SSL_CERT_FILE_${id}`],
-      sslCertFilePassword: process.env[`SSL_CERT_FILE_PASSWORD_${id}`],
-      sslKeyFile: process.env[`SSL_KEY_FILE_${id}`],
-      sslRejectUnauthorized: process.env[`SSL_REJECT_UNAUTHORIZED_${id}`],
-      trustServerCertificate: process.env[`SSL_TRUST_CERTIFICATE_${id}`],
-    }));
+    const connections = extractConnectionsFromEnv(process.env);
 
     for (const conn of connections) {
       for (const prop in process.env) {
@@ -229,6 +184,15 @@ module.exports = {
       );
     }
     await this.checkUnsavedConnectionsLimit();
+
+    if (process.env.STORAGE_DATABASE && process.env.CONNECTIONS) {
+      const storage = require('./storage');
+      try {
+        await storage.fillStorageConnectionsFromEnv();
+      } catch (err) {
+        logger.error(extractErrorLogData(err), 'DBGM-00268 Error filling storage connections from env');
+      }
+    }
   },
 
   list_meta: true,
@@ -241,7 +205,7 @@ module.exports = {
       return storageConnections;
     }
     if (portalConnections) {
-      if (platformInfo.allowShellConnection) return portalConnections;
+      if (platformInfo.allowShellConnection) return portalConnections.map(x => encryptConnection(x));
       return portalConnections.map(maskConnection).filter(x => connectionHasPermission(x, loadedPermissions));
     }
     return (await this.datastore.find()).filter(x => connectionHasPermission(x, loadedPermissions));
@@ -277,14 +241,60 @@ module.exports = {
     );
     pipeForkLogs(subprocess);
     subprocess.send({ ...connection, requestDbList });
-    return new Promise(resolve => {
+    return new Promise((resolve, reject) => {
+      let isWaitingForVolatile = false;
+
+      const cleanup = () => {
+        if (connection._id && pendingTestSubprocesses[connection._id]) {
+          delete pendingTestSubprocesses[connection._id];
+        }
+      };
+
       subprocess.on('message', resp => {
         if (handleProcessCommunication(resp, subprocess)) return;
         // @ts-ignore
-        const { msgtype } = resp;
+        const { msgtype, missingCredentialsDetail } = resp;
         if (msgtype == 'connected' || msgtype == 'error') {
+          cleanup();
           resolve(resp);
         }
+        if (msgtype == 'missingCredentials') {
+          if (missingCredentialsDetail?.redirectToDbLogin) {
+            // Store the subprocess for later when volatile connection is ready
+            isWaitingForVolatile = true;
+            pendingTestSubprocesses[connection._id] = {
+              subprocess,
+              requestDbList,
+            };
+            // Return immediately with redirectToDbLogin status in the old format
+            resolve({
+              missingCredentials: true,
+              detail: {
+                ...missingCredentialsDetail,
+                keepErrorResponseFromApi: true,
+              },
+            });
+            return;
+          }
+          reject(new MissingCredentialsError(missingCredentialsDetail));
+        }
+      });
+
+      subprocess.on('exit', code => {
+        // If exit happens while waiting for volatile, that's expected
+        if (isWaitingForVolatile && code === 0) {
+          cleanup();
+          return;
+        }
+        cleanup();
+        if (code !== 0) {
+          reject(new Error(`Test subprocess exited with code ${code}`));
+        }
+      });
+
+      subprocess.on('error', err => {
+        cleanup();
+        reject(err);
       });
     });
   },
@@ -317,6 +327,38 @@ module.exports = {
       return testRes;
     } else {
       volatileConnections[res._id] = res;
+
+      // Check if there's a pending test subprocess waiting for this volatile connection
+      const pendingTest = pendingTestSubprocesses[conid];
+      if (pendingTest) {
+        const { subprocess, requestDbList } = pendingTest;
+        try {
+          // Send the volatile connection to the waiting subprocess
+          subprocess.send({ ...res, requestDbList, isVolatileResolved: true });
+
+          // Wait for the test result and emit it as an event
+          subprocess.once('message', resp => {
+            if (handleProcessCommunication(resp, subprocess)) return;
+            const { msgtype } = resp;
+            if (msgtype == 'connected' || msgtype == 'error') {
+              // Emit SSE event with test result
+              socket.emit(`connection-test-result-${conid}`, {
+                ...resp,
+                volatileConId: res._id,
+              });
+              delete pendingTestSubprocesses[conid];
+            }
+          });
+        } catch (err) {
+          logger.error(extractErrorLogData(err), 'DBGM-00118 Error sending volatile connection to test subprocess');
+          socket.emit(`connection-test-result-${conid}`, {
+            msgtype: 'error',
+            error: err.message,
+          });
+          delete pendingTestSubprocesses[conid];
+        }
+      }
+
       return res;
     }
   },
@@ -447,7 +489,7 @@ module.exports = {
 
     if (portalConnections) {
       const res = portalConnections.find(x => x._id == conid) || null;
-      return mask && !platformInfo.allowShellConnection ? maskConnection(res) : res;
+      return mask && !platformInfo.allowShellConnection ? maskConnection(res) : encryptConnection(res);
     }
     const res = await this.datastore.get(conid);
     return res || null;

packages/api/src/controllers/databaseConnections.js

@@ -393,6 +393,12 @@ module.exports = {
     return null;
   },
 
+  dispatchRedisKeysChanged_meta: true,
+  dispatchRedisKeysChanged({ conid, database }) {
+    socket.emit(`redis-keys-changed-${conid}-${database}`);
+    return null;
+  },
+
   loadKeys_meta: true,
   async loadKeys({ conid, database, root, filter, limit }, req) {
     await testConnectionPermission(conid, req);
@@ -494,6 +500,20 @@ module.exports = {
     return res.result || null;
   },
 
+  multiCallMethod_meta: true,
+  async multiCallMethod({ conid, database, callList }, req) {
+    await testConnectionPermission(conid, req);
+
+    const opened = await this.ensureOpened(conid, database);
+    const res = await this.sendRequest(opened, { msgtype: 'multiCallMethod', callList });
+    if (res.errorMessage) {
+      return {
+        errorMessage: res.errorMessage,
+      };
+    }
+    return res.result || null;
+  },
+
   status_meta: true,
   async status({ conid, database }, req) {
     if (!conid) {

packages/api/src/controllers/files.js

@@ -68,6 +68,7 @@ module.exports = {
     await fs.unlink(path.join(filesdir(), folder, file));
     socket.emitChanged(`files-changed`, { folder });
     socket.emitChanged(`all-files-changed`);
+    this.emitChangedFolder(folder);
     return true;
   },
 
@@ -140,6 +141,15 @@ module.exports = {
     return deserialize(format, text);
   },
 
+  emitChangedFolder(folder) {
+    if (folder == 'themes') {
+      socket.emitChanged(`file-themes-changed`);
+    }
+    if (folder == 'favorites') {
+      socket.emitChanged('files-changed-favorites');
+    }
+  },
+
   save_meta: true,
   async save({ folder, file, data, format }, req) {
     const loadedPermissions = await loadPermissionsFromRequest(req);
@@ -173,6 +183,8 @@ module.exports = {
       if (folder == 'shell') {
         scheduler.reload();
       }
+      this.emitChangedFolder(folder);
+
       return true;
     }
   },
@@ -240,8 +252,8 @@ module.exports = {
   },
 
   exportDiagram_meta: true,
-  async exportDiagram({ filePath, html, css, themeType, themeClassName, watermark }) {
-    await fs.writeFile(filePath, getDiagramExport(html, css, themeType, themeClassName, watermark));
+  async exportDiagram({ filePath, html, css, themeType, themeVariables, watermark }) {
+    await fs.writeFile(filePath, getDiagramExport(html, css, themeType, themeVariables, watermark));
     return true;
   },
 
@@ -346,4 +358,20 @@ module.exports = {
     }
     return res;
   },
+
+  getFileThemes_meta: true,
+  async getFileThemes(_params, req) {
+    const loadedPermissions = await loadPermissionsFromRequest(req);
+    if (!hasPermission(`files/themes/read`, loadedPermissions)) return [];
+    const dir = path.join(filesdir(), 'themes');
+    if (!(await fs.exists(dir))) return [];
+    const files = await fs.readdir(dir);
+    const res = [];
+    for (const file of files) {
+      const filePath = path.join(dir, file);
+      const text = await fs.readFile(filePath, { encoding: 'utf-8' });
+      res.push(JSON.parse(text));
+    }
+    return res;
+  },
 };

packages/api/src/currentVersion.js

@@ -1,5 +1,5 @@
 
 module.exports = { 
-  version: '6.0.0-alpha.1',
+  version: '7.0.0-alpha.1',
   buildTime: '2024-12-01T00:00:00Z'
 };

packages/api/src/index.js

@@ -147,6 +147,7 @@ const shell = require('./shell/index');
 global.DBGATE_PACKAGES = {
   'dbgate-tools': require('dbgate-tools'),
   'dbgate-sqltree': require('dbgate-sqltree'),
+  'dbgate-datalib': require('dbgate-datalib'),
 };
 
 if (processArgs.startProcess) {

packages/api/src/proc/connectProcess.js

@@ -18,13 +18,36 @@ Platform: ${process.platform}
 
 function start() {
   childProcessChecker();
-  process.on('message', async connection => {
+
+  let isWaitingForVolatile = false;
+
+  const handleConnection = async connection => {
     // @ts-ignore
     const { requestDbList } = connection;
     if (handleProcessCommunication(connection)) return;
+
     try {
       const driver = requireEngineDriver(connection);
-      const dbhan = await connectUtility(driver, connection, 'app');
+      const connectionChanged = driver?.beforeConnectionSave ? driver.beforeConnectionSave(connection) : connection;
+
+      if (!connection.isVolatileResolved) {
+        if (connectionChanged.useRedirectDbLogin) {
+          process.send({
+            msgtype: 'missingCredentials',
+            missingCredentialsDetail: {
+              // @ts-ignore
+              conid: connection._id,
+              redirectToDbLogin: true,
+              keepErrorResponseFromApi: true,
+            },
+          });
+          // Don't exit - wait for volatile connection to be sent
+          isWaitingForVolatile = true;
+          return;
+        }
+      }
+
+      const dbhan = await connectUtility(driver, connectionChanged, 'app');
       let version = {
         version: 'Unknown',
       };
@@ -45,6 +68,16 @@ function start() {
     }
 
     process.exit(0);
+  };
+
+  process.on('message', async connection => {
+    // If we're waiting for volatile and receive a new connection, use it
+    if (isWaitingForVolatile) {
+      isWaitingForVolatile = false;
+      await handleConnection(connection);
+    } else {
+      await handleConnection(connection);
+    }
   });
 }
 

packages/api/src/proc/databaseConnectionProcess.js

@@ -368,6 +368,107 @@ async function handleSaveTableData({ msgid, changeSet }) {
   }
 }
 
+async function handleMultiCallMethod({ msgid, callList }) {
+  try {
+    const driver = requireEngineDriver(storedConnection);
+    await driver.invokeMethodCallList(dbhan, callList);
+
+    // for (const change of changeSet.changes) {
+    //   if (change.type === 'string') {
+    //     await driver.query(dbhan, `SET "${change.key}" "${change.value}"`);
+    //   } else if (change.type === 'json') {
+    //     await driver.query(dbhan, `JSON.SET "${change.key}" $ '${change.value.replace(/'/g, "\\'")}'`);
+    //   } else if (change.type === 'hash') {
+    //     if (change.updates && Array.isArray(change.updates)) {
+    //       for (const update of change.updates) {
+    //         await driver.query(dbhan, `HSET "${change.key}" "${update.key}" "${update.value}"`);
+          
+    //         if (update.ttl !== undefined && update.ttl !== null && update.ttl !== -1) {
+    //           try {
+    //             await dbhan.client.call('HEXPIRE', change.key, update.ttl, 'FIELDS', 1, update.key);
+    //           } catch (e) {}
+    //         }
+    //       }
+    //     }
+    //     if (change.inserts && Array.isArray(change.inserts)) {
+    //       for (const insert of change.inserts) {
+    //         await driver.query(dbhan, `HSET "${change.key}" "${insert.key}" "${insert.value}"`);
+            
+    //         if (insert.ttl !== undefined && insert.ttl !== null && insert.ttl !== -1) {
+    //           try {
+    //             await dbhan.client.call('HEXPIRE', change.key, insert.ttl, 'FIELDS', 1, insert.key);
+    //           } catch (e) {}
+    //         }
+    //       }
+    //     }
+    //     if (change.deletes && Array.isArray(change.deletes)) {
+    //       for (const delKey of change.deletes) {
+    //         await driver.query(dbhan, `HDEL "${change.key}" "${delKey}"`);
+    //       }
+    //     }
+    //   } else if (change.type === 'zset') {
+    //     if (change.updates && Array.isArray(change.updates)) {
+    //       for (const update of change.updates) {
+    //         await driver.query(dbhan, `ZADD "${change.key}" ${update.score} "${update.member}"`);
+    //       }
+    //     }
+    //     if (change.inserts && Array.isArray(change.inserts)) {
+    //       for (const insert of change.inserts) {
+    //         await driver.query(dbhan, `ZADD "${change.key}" ${insert.score} "${insert.member}"`);
+    //       }
+    //     }
+    //     if (change.deletes && Array.isArray(change.deletes)) {
+    //       for (const delMember of change.deletes) {
+    //         await driver.query(dbhan, `ZREM "${change.key}" "${delMember}"`);
+    //       }
+    //     }
+    //   } else if (change.type === 'list') {
+    //     if (change.updates && Array.isArray(change.updates)) {
+    //       for (const update of change.updates) {
+    //         await driver.query(dbhan, `LSET "${change.key}" ${update.index} "${update.value}"`);
+    //       }
+    //     }
+    //     if (change.inserts && Array.isArray(change.inserts)) {
+    //       for (const insert of change.inserts) {
+    //         await driver.query(dbhan, `RPUSH "${change.key}" "${insert.value}"`);
+    //       }
+    //     }
+    //   } else if (change.type === 'set') {
+    //     if (change.inserts && Array.isArray(change.inserts)) {
+    //       for (const insert of change.inserts) {
+    //         await driver.query(dbhan, `SADD "${change.key}" "${insert.value}"`);
+    //       }
+    //     }
+    //     if (change.deletes && Array.isArray(change.deletes)) {
+    //       for (const delValue of change.deletes) {
+    //         await driver.query(dbhan, `SREM "${change.key}" "${delValue}"`);
+    //       }
+    //     }
+    //   } else if (change.type === 'stream') {
+    //     if (change.inserts && Array.isArray(change.inserts)) {
+    //       for (const insert of change.inserts) {
+    //         const streamId = insert.id === '*' || !insert.id ? '*' : insert.id;
+    //         await driver.query(dbhan, `XADD "${change.key}" ${streamId} value "${insert.value}"`);
+    //       }
+    //     }
+    //     if (change.deletes && Array.isArray(change.deletes)) {
+    //       for (const delId of change.deletes) {
+    //         await driver.query(dbhan, `XDEL "${change.key}" "${delId}"`);
+    //       }
+    //     }
+    //   }
+    // }
+
+    process.send({ msgtype: 'response', msgid });
+  } catch (err) {
+    process.send({
+      msgtype: 'response',
+      msgid,
+      errorMessage: extractErrorMessage(err, 'Error saving Redis data'),
+    });
+  }
+}
+
 async function handleSqlPreview({ msgid, objects, options }) {
   await waitStructure();
   const driver = requireEngineDriver(storedConnection);
@@ -501,6 +602,7 @@ const messageHandlers = {
   schemaList: handleSchemaList,
   executeSessionQuery: handleExecuteSessionQuery,
   evalJsonScript: handleEvalJsonScript,
+  multiCallMethod: handleMultiCallMethod,
   // runCommand: handleRunCommand,
 };
 

packages/api/src/shell/copyStream.js

@@ -65,6 +65,8 @@ async function copyStream(input, output, options) {
       });
     }
   } catch (err) {
+    logger.error(extractErrorLogData(err, { progressName }), 'DBGM-00157 Import/export job failed');
+
     process.send({
       msgtype: 'copyStreamError',
       copyStreamError: {
@@ -82,8 +84,6 @@ async function copyStream(input, output, options) {
         errorMessage: extractErrorMessage(err),
       });
     }
-
-    logger.error(extractErrorLogData(err, { progressName }), 'DBGM-00157 Import/export job failed');
     // throw err;
   }
 }

packages/api/src/shell/dataReplicator.js

@@ -64,6 +64,7 @@ async function dataReplicator({
           createNew: compileOperationFunction(item.createNew, item.createCondition),
           updateExisting: compileOperationFunction(item.updateExisting, item.updateCondition),
           deleteMissing: !!item.deleteMissing,
+          skipUpdateColumns: item.skipUpdateColumns,
           deleteRestrictionColumns: item.deleteRestrictionColumns ?? [],
           openStream: item.openStream
             ? item.openStream

packages/api/src/storageModel.js

@@ -686,9 +686,34 @@ module.exports = {
           "columnName": "connectionDefinition",
           "dataType": "text",
           "notNull": false
+        },
+        {
+          "pureName": "connections",
+          "columnName": "import_source_id",
+          "dataType": "int",
+          "notNull": false
+        },
+        {
+          "pureName": "connections",
+          "columnName": "id_original",
+          "dataType": "varchar(250)",
+          "notNull": false
+        }
+      ],
+      "foreignKeys": [
+        {
+          "constraintType": "foreignKey",
+          "constraintName": "FK_connections_import_source_id",
+          "pureName": "connections",
+          "refTableName": "import_sources",
+          "columns": [
+            {
+              "columnName": "import_source_id",
+              "refColumnName": "id"
+            }
+          ]
         }
       ],
-      "foreignKeys": [],
       "primaryKey": {
         "pureName": "connections",
         "constraintType": "primaryKey",
@@ -790,6 +815,119 @@ module.exports = {
         }
       ]
     },
+    {
+      "pureName": "import_sources",
+      "columns": [
+        {
+          "pureName": "import_sources",
+          "columnName": "id",
+          "dataType": "int",
+          "autoIncrement": true,
+          "notNull": true
+        },
+        {
+          "pureName": "import_sources",
+          "columnName": "name",
+          "dataType": "varchar(250)",
+          "notNull": true
+        }
+      ],
+      "foreignKeys": [],
+      "primaryKey": {
+        "pureName": "import_sources",
+        "constraintType": "primaryKey",
+        "constraintName": "PK_import_sources",
+        "columns": [
+          {
+            "columnName": "id"
+          }
+        ]
+      },
+      "preloadedRows": [
+        {
+          "id": -1,
+          "name": "env"
+        }
+      ]
+    },
+    {
+      "pureName": "password_reset_tokens",
+      "columns": [
+        {
+          "pureName": "password_reset_tokens",
+          "columnName": "id",
+          "dataType": "int",
+          "autoIncrement": true,
+          "notNull": true
+        },
+        {
+          "pureName": "password_reset_tokens",
+          "columnName": "user_id",
+          "dataType": "int",
+          "notNull": true
+        },
+        {
+          "pureName": "password_reset_tokens",
+          "columnName": "token",
+          "dataType": "varchar(500)",
+          "notNull": true
+        },
+        {
+          "pureName": "password_reset_tokens",
+          "columnName": "created_at",
+          "dataType": "datetime",
+          "notNull": true
+        },
+        {
+          "pureName": "password_reset_tokens",
+          "columnName": "expires_at",
+          "dataType": "datetime",
+          "notNull": true
+        },
+        {
+          "pureName": "password_reset_tokens",
+          "columnName": "used_at",
+          "dataType": "datetime",
+          "notNull": false
+        }
+      ],
+      "foreignKeys": [
+        {
+          "constraintType": "foreignKey",
+          "constraintName": "FK_password_reset_tokens_user_id",
+          "pureName": "password_reset_tokens",
+          "refTableName": "users",
+          "columns": [
+            {
+              "columnName": "user_id",
+              "refColumnName": "id"
+            }
+          ]
+        }
+      ],
+      "indexes": [
+        {
+          "constraintName": "idx_token",
+          "pureName": "password_reset_tokens",
+          "constraintType": "index",
+          "columns": [
+            {
+              "columnName": "token"
+            }
+          ]
+        }
+      ],
+      "primaryKey": {
+        "pureName": "password_reset_tokens",
+        "constraintType": "primaryKey",
+        "constraintName": "PK_password_reset_tokens",
+        "columns": [
+          {
+            "columnName": "id"
+          }
+        ]
+      }
+    },
     {
       "pureName": "roles",
       "columns": [
@@ -805,9 +943,34 @@ module.exports = {
           "columnName": "name",
           "dataType": "varchar(250)",
           "notNull": false
+        },
+        {
+          "pureName": "roles",
+          "columnName": "import_source_id",
+          "dataType": "int",
+          "notNull": false
+        },
+        {
+          "pureName": "roles",
+          "columnName": "id_original",
+          "dataType": "varchar(250)",
+          "notNull": false
+        }
+      ],
+      "foreignKeys": [
+        {
+          "constraintType": "foreignKey",
+          "constraintName": "FK_roles_import_source_id",
+          "pureName": "roles",
+          "refTableName": "import_sources",
+          "columns": [
+            {
+              "columnName": "import_source_id",
+              "refColumnName": "id"
+            }
+          ]
         }
       ],
-      "foreignKeys": [],
       "primaryKey": {
         "pureName": "roles",
         "constraintType": "primaryKey",
@@ -854,6 +1017,12 @@ module.exports = {
           "columnName": "connection_id",
           "dataType": "int",
           "notNull": true
+        },
+        {
+          "pureName": "role_connections",
+          "columnName": "import_source_id",
+          "dataType": "int",
+          "notNull": false
         }
       ],
       "foreignKeys": [
@@ -882,6 +1051,18 @@ module.exports = {
               "refColumnName": "id"
             }
           ]
+        },
+        {
+          "constraintType": "foreignKey",
+          "constraintName": "FK_role_connections_import_source_id",
+          "pureName": "role_connections",
+          "refTableName": "import_sources",
+          "columns": [
+            {
+              "columnName": "import_source_id",
+              "refColumnName": "id"
+            }
+          ]
         }
       ],
       "primaryKey": {
@@ -934,6 +1115,18 @@ module.exports = {
           "columnName": "database_permission_role_id",
           "dataType": "int",
           "notNull": true
+        },
+        {
+          "pureName": "role_databases",
+          "columnName": "import_source_id",
+          "dataType": "int",
+          "notNull": false
+        },
+        {
+          "pureName": "role_databases",
+          "columnName": "id_original",
+          "dataType": "varchar(250)",
+          "notNull": false
         }
       ],
       "foreignKeys": [
@@ -974,6 +1167,18 @@ module.exports = {
               "refColumnName": "id"
             }
           ]
+        },
+        {
+          "constraintType": "foreignKey",
+          "constraintName": "FK_role_databases_import_source_id",
+          "pureName": "role_databases",
+          "refTableName": "import_sources",
+          "columns": [
+            {
+              "columnName": "import_source_id",
+              "refColumnName": "id"
+            }
+          ]
         }
       ],
       "primaryKey": {
@@ -1087,6 +1292,12 @@ module.exports = {
           "columnName": "permission",
           "dataType": "varchar(250)",
           "notNull": true
+        },
+        {
+          "pureName": "role_permissions",
+          "columnName": "import_source_id",
+          "dataType": "int",
+          "notNull": false
         }
       ],
       "foreignKeys": [
@@ -1102,6 +1313,18 @@ module.exports = {
               "refColumnName": "id"
             }
           ]
+        },
+        {
+          "constraintType": "foreignKey",
+          "constraintName": "FK_role_permissions_import_source_id",
+          "pureName": "role_permissions",
+          "refTableName": "import_sources",
+          "columns": [
+            {
+              "columnName": "import_source_id",
+              "refColumnName": "id"
+            }
+          ]
         }
       ],
       "primaryKey": {
@@ -1184,6 +1407,18 @@ module.exports = {
           "columnName": "table_permission_scope_id",
           "dataType": "int",
           "notNull": true
+        },
+        {
+          "pureName": "role_tables",
+          "columnName": "import_source_id",
+          "dataType": "int",
+          "notNull": false
+        },
+        {
+          "pureName": "role_tables",
+          "columnName": "id_original",
+          "dataType": "varchar(250)",
+          "notNull": false
         }
       ],
       "foreignKeys": [
@@ -1236,6 +1471,18 @@ module.exports = {
               "refColumnName": "id"
             }
           ]
+        },
+        {
+          "constraintType": "foreignKey",
+          "constraintName": "FK_role_tables_import_source_id",
+          "pureName": "role_tables",
+          "refTableName": "import_sources",
+          "columns": [
+            {
+              "columnName": "import_source_id",
+              "refColumnName": "id"
+            }
+          ]
         }
       ],
       "primaryKey": {
@@ -1329,6 +1576,86 @@ module.exports = {
         ]
       }
     },
+    {
+      "pureName": "role_team_folders",
+      "columns": [
+        {
+          "pureName": "role_team_folders",
+          "columnName": "id",
+          "dataType": "int",
+          "autoIncrement": true,
+          "notNull": true
+        },
+        {
+          "pureName": "role_team_folders",
+          "columnName": "role_id",
+          "dataType": "int",
+          "notNull": true
+        },
+        {
+          "pureName": "role_team_folders",
+          "columnName": "team_folder_id",
+          "dataType": "int",
+          "notNull": true
+        },
+        {
+          "pureName": "role_team_folders",
+          "columnName": "allow_read_files",
+          "dataType": "int",
+          "notNull": false
+        },
+        {
+          "pureName": "role_team_folders",
+          "columnName": "allow_write_files",
+          "dataType": "int",
+          "notNull": false
+        },
+        {
+          "pureName": "role_team_folders",
+          "columnName": "allow_use_files",
+          "dataType": "int",
+          "notNull": false
+        }
+      ],
+      "foreignKeys": [
+        {
+          "constraintType": "foreignKey",
+          "constraintName": "FK_role_team_folders_role_id",
+          "pureName": "role_team_folders",
+          "refTableName": "roles",
+          "deleteAction": "CASCADE",
+          "columns": [
+            {
+              "columnName": "role_id",
+              "refColumnName": "id"
+            }
+          ]
+        },
+        {
+          "constraintType": "foreignKey",
+          "constraintName": "FK_role_team_folders_team_folder_id",
+          "pureName": "role_team_folders",
+          "refTableName": "team_folders",
+          "deleteAction": "CASCADE",
+          "columns": [
+            {
+              "columnName": "team_folder_id",
+              "refColumnName": "id"
+            }
+          ]
+        }
+      ],
+      "primaryKey": {
+        "pureName": "role_team_folders",
+        "constraintType": "primaryKey",
+        "constraintName": "PK_role_team_folders",
+        "columns": [
+          {
+            "columnName": "id"
+          }
+        ]
+      }
+    },
     {
       "pureName": "table_permission_roles",
       "columns": [
@@ -1486,6 +1813,14 @@ module.exports = {
           "columnName": "metadata",
           "dataType": "varchar(1000)",
           "notNull": false
+        },
+        {
+          "pureName": "team_files",
+          "columnName": "team_folder_id",
+          "dataType": "int",
+          "notNull": true,
+          "defaultValue": -1,
+          "defaultConstraint": "DF_team_files_team_folder_id"
         }
       ],
       "foreignKeys": [
@@ -1512,6 +1847,18 @@ module.exports = {
               "refColumnName": "id"
             }
           ]
+        },
+        {
+          "constraintType": "foreignKey",
+          "constraintName": "FK_team_files_team_folder_id",
+          "pureName": "team_files",
+          "refTableName": "team_folders",
+          "columns": [
+            {
+              "columnName": "team_folder_id",
+              "refColumnName": "id"
+            }
+          ]
         }
       ],
       "primaryKey": {
@@ -1596,6 +1943,41 @@ module.exports = {
         }
       ]
     },
+    {
+      "pureName": "team_folders",
+      "columns": [
+        {
+          "pureName": "team_folders",
+          "columnName": "id",
+          "dataType": "int",
+          "autoIncrement": true,
+          "notNull": true
+        },
+        {
+          "pureName": "team_folders",
+          "columnName": "folder_name",
+          "dataType": "varchar(250)",
+          "notNull": false
+        }
+      ],
+      "foreignKeys": [],
+      "primaryKey": {
+        "pureName": "team_folders",
+        "constraintType": "primaryKey",
+        "constraintName": "PK_team_folders",
+        "columns": [
+          {
+            "columnName": "id"
+          }
+        ]
+      },
+      "preloadedRows": [
+        {
+          "id": -1,
+          "folder_name": "default"
+        }
+      ]
+    },
     {
       "pureName": "users",
       "columns": [
@@ -2194,6 +2576,86 @@ module.exports = {
           }
         ]
       }
+    },
+    {
+      "pureName": "user_team_folders",
+      "columns": [
+        {
+          "pureName": "user_team_folders",
+          "columnName": "id",
+          "dataType": "int",
+          "autoIncrement": true,
+          "notNull": true
+        },
+        {
+          "pureName": "user_team_folders",
+          "columnName": "user_id",
+          "dataType": "int",
+          "notNull": true
+        },
+        {
+          "pureName": "user_team_folders",
+          "columnName": "team_folder_id",
+          "dataType": "int",
+          "notNull": true
+        },
+        {
+          "pureName": "user_team_folders",
+          "columnName": "allow_read_files",
+          "dataType": "int",
+          "notNull": false
+        },
+        {
+          "pureName": "user_team_folders",
+          "columnName": "allow_write_files",
+          "dataType": "int",
+          "notNull": false
+        },
+        {
+          "pureName": "user_team_folders",
+          "columnName": "allow_use_files",
+          "dataType": "int",
+          "notNull": false
+        }
+      ],
+      "foreignKeys": [
+        {
+          "constraintType": "foreignKey",
+          "constraintName": "FK_user_team_folders_user_id",
+          "pureName": "user_team_folders",
+          "refTableName": "users",
+          "deleteAction": "CASCADE",
+          "columns": [
+            {
+              "columnName": "user_id",
+              "refColumnName": "id"
+            }
+          ]
+        },
+        {
+          "constraintType": "foreignKey",
+          "constraintName": "FK_user_team_folders_team_folder_id",
+          "pureName": "user_team_folders",
+          "refTableName": "team_folders",
+          "deleteAction": "CASCADE",
+          "columns": [
+            {
+              "columnName": "team_folder_id",
+              "refColumnName": "id"
+            }
+          ]
+        }
+      ],
+      "primaryKey": {
+        "pureName": "user_team_folders",
+        "constraintType": "primaryKey",
+        "constraintName": "PK_user_team_folders",
+        "columns": [
+          {
+            "columnName": "id"
+          }
+        ]
+      }
     }
   ],
   "collections": [],

packages/api/src/utility/envtools.js

@@ -0,0 +1,454 @@
+const path = require('path');
+const _ = require('lodash');
+const { safeJsonParse, getDatabaseFileLabel } = require('dbgate-tools');
+const crypto = require('crypto');
+
+function extractConnectionsFromEnv(env) {
+  if (!env?.CONNECTIONS) {
+    return null;
+  }
+
+  const connections = _.compact(env.CONNECTIONS.split(',')).map(id => ({
+    _id: id,
+    engine: env[`ENGINE_${id}`],
+    server: env[`SERVER_${id}`],
+    user: env[`USER_${id}`],
+    password: env[`PASSWORD_${id}`],
+    passwordMode: env[`PASSWORD_MODE_${id}`],
+    port: env[`PORT_${id}`],
+    databaseUrl: env[`URL_${id}`],
+    useDatabaseUrl: !!env[`URL_${id}`],
+    databaseFile: env[`FILE_${id}`]?.replace(
+      '%%E2E_TEST_DATA_DIRECTORY%%',
+      path.join(path.dirname(path.dirname(__dirname)), 'e2e-tests', 'tmpdata')
+    ),
+    socketPath: env[`SOCKET_PATH_${id}`],
+    serviceName: env[`SERVICE_NAME_${id}`],
+    authType: env[`AUTH_TYPE_${id}`] || (env[`SOCKET_PATH_${id}`] ? 'socket' : undefined),
+    defaultDatabase: env[`DATABASE_${id}`] || (env[`FILE_${id}`] ? getDatabaseFileLabel(env[`FILE_${id}`]) : null),
+    singleDatabase: !!env[`DATABASE_${id}`] || !!env[`FILE_${id}`],
+    displayName: env[`LABEL_${id}`],
+    isReadOnly: env[`READONLY_${id}`],
+    databases: env[`DBCONFIG_${id}`] ? safeJsonParse(env[`DBCONFIG_${id}`]) : null,
+    allowedDatabases: env[`ALLOWED_DATABASES_${id}`]?.replace(/\|/g, '\n'),
+    allowedDatabasesRegex: env[`ALLOWED_DATABASES_REGEX_${id}`],
+    parent: env[`PARENT_${id}`] || undefined,
+    useSeparateSchemas: !!env[`USE_SEPARATE_SCHEMAS_${id}`],
+    localDataCenter: env[`LOCAL_DATA_CENTER_${id}`],
+
+    // SSH tunnel
+    useSshTunnel: env[`USE_SSH_${id}`],
+    sshHost: env[`SSH_HOST_${id}`],
+    sshPort: env[`SSH_PORT_${id}`],
+    sshMode: env[`SSH_MODE_${id}`],
+    sshLogin: env[`SSH_LOGIN_${id}`],
+    sshPassword: env[`SSH_PASSWORD_${id}`],
+    sshKeyfile: env[`SSH_KEY_FILE_${id}`],
+    sshKeyfilePassword: env[`SSH_KEY_FILE_PASSWORD_${id}`],
+
+    // SSL
+    useSsl: env[`USE_SSL_${id}`],
+    sslCaFile: env[`SSL_CA_FILE_${id}`],
+    sslCertFile: env[`SSL_CERT_FILE_${id}`],
+    sslCertFilePassword: env[`SSL_CERT_FILE_PASSWORD_${id}`],
+    sslKeyFile: env[`SSL_KEY_FILE_${id}`],
+    sslRejectUnauthorized: env[`SSL_REJECT_UNAUTHORIZED_${id}`],
+    trustServerCertificate: env[`SSL_TRUST_CERTIFICATE_${id}`],
+  }));
+
+  return connections;
+}
+
+function extractImportEntitiesFromEnv(env) {
+  const portalConnections = extractConnectionsFromEnv(env) || [];
+
+  const connections = portalConnections.map((conn, index) => ({
+    ...conn,
+    id_original: conn._id,
+    import_source_id: -1,
+    conid: crypto.randomUUID(),
+    _id: undefined,
+    id: index + 1, // autoincrement id
+
+    useDatabaseUrl: conn.useDatabaseUrl ? 1 : 0,
+    isReadOnly: conn.isReadOnly ? 1 : 0,
+    useSeparateSchemas: conn.useSeparateSchemas ? 1 : 0,
+    trustServerCertificate: conn.trustServerCertificate ? 1 : 0,
+    singleDatabase: conn.singleDatabase ? 1 : 0,
+    useSshTunnel: conn.useSshTunnel ? 1 : 0,
+    useSsl: conn.useSsl ? 1 : 0,
+    sslRejectUnauthorized: conn.sslRejectUnauthorized ? 1 : 0,
+  }));
+
+  const connectionEnvIdToDbId = {};
+  for (const conn of connections) {
+    connectionEnvIdToDbId[conn.id_original] = conn.id;
+  }
+
+  const connectionsRegex = /^ROLE_(.+)_CONNECTIONS$/;
+  const permissionsRegex = /^ROLE_(.+)_PERMISSIONS$/;
+
+  const dbConnectionRegex = /^ROLE_(.+)_DATABASES_(.+)_CONNECTION$/;
+  const dbDatabasesRegex = /^ROLE_(.+)_DATABASES_(.+)_DATABASES$/;
+  const dbDatabasesRegexRegex = /^ROLE_(.+)_DATABASES_(.+)_DATABASES_REGEX$/;
+  const dbPermissionRegex = /^ROLE_(.+)_DATABASES_(.+)_PERMISSION$/;
+
+  const tableConnectionRegex = /^ROLE_(.+)_TABLES_(.+)_CONNECTION$/;
+  const tableDatabasesRegex = /^ROLE_(.+)_TABLES_(.+)_DATABASES$/;
+  const tableDatabasesRegexRegex = /^ROLE_(.+)_TABLES_(.+)_DATABASES_REGEX$/;
+  const tableSchemasRegex = /^ROLE_(.+)_TABLES_(.+)_SCHEMAS$/;
+  const tableSchemasRegexRegex = /^ROLE_(.+)_TABLES_(.+)_SCHEMAS_REGEX$/;
+  const tableTablesRegex = /^ROLE_(.+)_TABLES_(.+)_TABLES$/;
+  const tableTablesRegexRegex = /^ROLE_(.+)_TABLES_(.+)_TABLES_REGEX$/;
+  const tablePermissionRegex = /^ROLE_(.+)_TABLES_(.+)_PERMISSION$/;
+  const tableScopeRegex = /^ROLE_(.+)_TABLES_(.+)_SCOPE$/;
+
+  const roles = [];
+  const role_connections = [];
+  const role_permissions = [];
+  const role_databases = [];
+  const role_tables = [];
+
+  // Permission name to ID mappings
+  const databasePermissionMap = {
+    view: -1,
+    read_content: -2,
+    write_data: -3,
+    run_script: -4,
+    deny: -5,
+  };
+
+  const tablePermissionMap = {
+    read: -1,
+    update_only: -2,
+    create_update_delete: -3,
+    run_script: -4,
+    deny: -5,
+  };
+
+  const tableScopeMap = {
+    all_objects: -1,
+    tables: -2,
+    views: -3,
+    tables_views_collections: -4,
+    procedures: -5,
+    functions: -6,
+    triggers: -7,
+    sql_objects: -8,
+    collections: -9,
+  };
+
+  // Collect database and table permissions data
+  const databasePermissions = {};
+  const tablePermissions = {};
+
+  // First pass: collect all database and table permission data
+  for (const key in env) {
+    const dbConnMatch = key.match(dbConnectionRegex);
+    const dbDatabasesMatch = key.match(dbDatabasesRegex);
+    const dbDatabasesRegexMatch = key.match(dbDatabasesRegexRegex);
+    const dbPermMatch = key.match(dbPermissionRegex);
+
+    const tableConnMatch = key.match(tableConnectionRegex);
+    const tableDatabasesMatch = key.match(tableDatabasesRegex);
+    const tableDatabasesRegexMatch = key.match(tableDatabasesRegexRegex);
+    const tableSchemasMatch = key.match(tableSchemasRegex);
+    const tableSchemasRegexMatch = key.match(tableSchemasRegexRegex);
+    const tableTablesMatch = key.match(tableTablesRegex);
+    const tableTablesRegexMatch = key.match(tableTablesRegexRegex);
+    const tablePermMatch = key.match(tablePermissionRegex);
+    const tableScopeMatch = key.match(tableScopeRegex);
+
+    // Database permissions
+    if (dbConnMatch) {
+      const [, roleName, permId] = dbConnMatch;
+      if (!databasePermissions[roleName]) databasePermissions[roleName] = {};
+      if (!databasePermissions[roleName][permId]) databasePermissions[roleName][permId] = {};
+      databasePermissions[roleName][permId].connection = env[key];
+    }
+    if (dbDatabasesMatch) {
+      const [, roleName, permId] = dbDatabasesMatch;
+      if (!databasePermissions[roleName]) databasePermissions[roleName] = {};
+      if (!databasePermissions[roleName][permId]) databasePermissions[roleName][permId] = {};
+      databasePermissions[roleName][permId].databases = env[key]?.replace(/\|/g, '\n');
+    }
+    if (dbDatabasesRegexMatch) {
+      const [, roleName, permId] = dbDatabasesRegexMatch;
+      if (!databasePermissions[roleName]) databasePermissions[roleName] = {};
+      if (!databasePermissions[roleName][permId]) databasePermissions[roleName][permId] = {};
+      databasePermissions[roleName][permId].databasesRegex = env[key];
+    }
+    if (dbPermMatch) {
+      const [, roleName, permId] = dbPermMatch;
+      if (!databasePermissions[roleName]) databasePermissions[roleName] = {};
+      if (!databasePermissions[roleName][permId]) databasePermissions[roleName][permId] = {};
+      databasePermissions[roleName][permId].permission = env[key];
+    }
+
+    // Table permissions
+    if (tableConnMatch) {
+      const [, roleName, permId] = tableConnMatch;
+      if (!tablePermissions[roleName]) tablePermissions[roleName] = {};
+      if (!tablePermissions[roleName][permId]) tablePermissions[roleName][permId] = {};
+      tablePermissions[roleName][permId].connection = env[key];
+    }
+    if (tableDatabasesMatch) {
+      const [, roleName, permId] = tableDatabasesMatch;
+      if (!tablePermissions[roleName]) tablePermissions[roleName] = {};
+      if (!tablePermissions[roleName][permId]) tablePermissions[roleName][permId] = {};
+      tablePermissions[roleName][permId].databases = env[key]?.replace(/\|/g, '\n');
+    }
+    if (tableDatabasesRegexMatch) {
+      const [, roleName, permId] = tableDatabasesRegexMatch;
+      if (!tablePermissions[roleName]) tablePermissions[roleName] = {};
+      if (!tablePermissions[roleName][permId]) tablePermissions[roleName][permId] = {};
+      tablePermissions[roleName][permId].databasesRegex = env[key];
+    }
+    if (tableSchemasMatch) {
+      const [, roleName, permId] = tableSchemasMatch;
+      if (!tablePermissions[roleName]) tablePermissions[roleName] = {};
+      if (!tablePermissions[roleName][permId]) tablePermissions[roleName][permId] = {};
+      tablePermissions[roleName][permId].schemas = env[key];
+    }
+    if (tableSchemasRegexMatch) {
+      const [, roleName, permId] = tableSchemasRegexMatch;
+      if (!tablePermissions[roleName]) tablePermissions[roleName] = {};
+      if (!tablePermissions[roleName][permId]) tablePermissions[roleName][permId] = {};
+      tablePermissions[roleName][permId].schemasRegex = env[key];
+    }
+    if (tableTablesMatch) {
+      const [, roleName, permId] = tableTablesMatch;
+      if (!tablePermissions[roleName]) tablePermissions[roleName] = {};
+      if (!tablePermissions[roleName][permId]) tablePermissions[roleName][permId] = {};
+      tablePermissions[roleName][permId].tables = env[key]?.replace(/\|/g, '\n');
+    }
+    if (tableTablesRegexMatch) {
+      const [, roleName, permId] = tableTablesRegexMatch;
+      if (!tablePermissions[roleName]) tablePermissions[roleName] = {};
+      if (!tablePermissions[roleName][permId]) tablePermissions[roleName][permId] = {};
+      tablePermissions[roleName][permId].tablesRegex = env[key];
+    }
+    if (tablePermMatch) {
+      const [, roleName, permId] = tablePermMatch;
+      if (!tablePermissions[roleName]) tablePermissions[roleName] = {};
+      if (!tablePermissions[roleName][permId]) tablePermissions[roleName][permId] = {};
+      tablePermissions[roleName][permId].permission = env[key];
+    }
+    if (tableScopeMatch) {
+      const [, roleName, permId] = tableScopeMatch;
+      if (!tablePermissions[roleName]) tablePermissions[roleName] = {};
+      if (!tablePermissions[roleName][permId]) tablePermissions[roleName][permId] = {};
+      tablePermissions[roleName][permId].scope = env[key];
+    }
+  }
+
+  // Second pass: process roles, connections, and permissions
+  for (const key in env) {
+    const connMatch = key.match(connectionsRegex);
+    const permMatch = key.match(permissionsRegex);
+    if (connMatch) {
+      const roleName = connMatch[1];
+      let role = roles.find(r => r.name === roleName);
+      if (!role) {
+        role = {
+          id: roles.length + 1,
+          name: roleName,
+          import_source_id: -1,
+        };
+        roles.push(role);
+      }
+      const connIds = env[key]
+        .split(',')
+        .map(id => id.trim())
+        .filter(id => id.length > 0);
+      for (const connId of connIds) {
+        const dbId = connectionEnvIdToDbId[connId];
+        if (dbId) {
+          role_connections.push({
+            role_id: role.id,
+            connection_id: dbId,
+            import_source_id: -1,
+          });
+        }
+      }
+    }
+    if (permMatch) {
+      const roleName = permMatch[1];
+      let role = roles.find(r => r.name === roleName);
+      if (!role) {
+        role = {
+          id: roles.length + 1,
+          name: roleName,
+          import_source_id: -1,
+        };
+        roles.push(role);
+      }
+      const permissions = env[key]
+        .split(',')
+        .map(p => p.trim())
+        .filter(p => p.length > 0);
+      for (const permission of permissions) {
+        role_permissions.push({
+          role_id: role.id,
+          permission,
+          import_source_id: -1,
+        });
+      }
+    }
+  }
+
+  // Process database permissions
+  for (const roleName in databasePermissions) {
+    let role = roles.find(r => r.name === roleName);
+    if (!role) {
+      role = {
+        id: roles.length + 1,
+        name: roleName,
+        import_source_id: -1,
+      };
+      roles.push(role);
+    }
+
+    for (const permId in databasePermissions[roleName]) {
+      const perm = databasePermissions[roleName][permId];
+      if (perm.connection && perm.permission) {
+        const dbId = connectionEnvIdToDbId[perm.connection];
+        const permissionId = databasePermissionMap[perm.permission];
+        if (dbId && permissionId) {
+          role_databases.push({
+            role_id: role.id,
+            connection_id: dbId,
+            database_names_list: perm.databases || null,
+            database_names_regex: perm.databasesRegex || null,
+            database_permission_role_id: permissionId,
+            id_original: permId,
+            import_source_id: -1,
+          });
+        }
+      }
+    }
+  }
+
+  // Process table permissions
+  for (const roleName in tablePermissions) {
+    let role = roles.find(r => r.name === roleName);
+    if (!role) {
+      role = {
+        id: roles.length + 1,
+        name: roleName,
+        import_source_id: -1,
+      };
+      roles.push(role);
+    }
+
+    for (const permId in tablePermissions[roleName]) {
+      const perm = tablePermissions[roleName][permId];
+      if (perm.connection && perm.permission) {
+        const dbId = connectionEnvIdToDbId[perm.connection];
+        const permissionId = tablePermissionMap[perm.permission];
+        const scopeId = tableScopeMap[perm.scope || 'all_objects'];
+        if (dbId && permissionId && scopeId) {
+          role_tables.push({
+            role_id: role.id,
+            connection_id: dbId,
+            database_names_list: perm.databases || null,
+            database_names_regex: perm.databasesRegex || null,
+            schema_names_list: perm.schemas || null,
+            schema_names_regex: perm.schemasRegex || null,
+            table_names_list: perm.tables || null,
+            table_names_regex: perm.tablesRegex || null,
+            table_permission_role_id: permissionId,
+            table_permission_scope_id: scopeId,
+            id_original: permId,
+            import_source_id: -1,
+          });
+        }
+      }
+    }
+  }
+
+  if (connections.length == 0 && roles.length == 0) {
+    return null;
+  }
+
+  return {
+    connections,
+    roles,
+    role_connections,
+    role_permissions,
+    role_databases,
+    role_tables,
+  };
+}
+
+function createStorageFromEnvReplicatorItems(importEntities) {
+  return [
+    {
+      name: 'connections',
+      findExisting: true,
+      createNew: true,
+      updateExisting: true,
+      matchColumns: ['id_original', 'import_source_id'],
+      deleteMissing: true,
+      deleteRestrictionColumns: ['import_source_id'],
+      skipUpdateColumns: ['conid'],
+      jsonArray: importEntities.connections,
+    },
+    {
+      name: 'roles',
+      findExisting: true,
+      createNew: true,
+      updateExisting: true,
+      matchColumns: ['name', 'import_source_id'],
+      deleteMissing: true,
+      deleteRestrictionColumns: ['import_source_id'],
+      jsonArray: importEntities.roles,
+    },
+    {
+      name: 'role_connections',
+      findExisting: true,
+      createNew: true,
+      updateExisting: false,
+      deleteMissing: true,
+      matchColumns: ['role_id', 'connection_id', 'import_source_id'],
+      jsonArray: importEntities.role_connections,
+      deleteRestrictionColumns: ['import_source_id'],
+    },
+    {
+      name: 'role_permissions',
+      findExisting: true,
+      createNew: true,
+      updateExisting: false,
+      deleteMissing: true,
+      matchColumns: ['role_id', 'permission', 'import_source_id'],
+      jsonArray: importEntities.role_permissions,
+      deleteRestrictionColumns: ['import_source_id'],
+    },
+    {
+      name: 'role_databases',
+      findExisting: true,
+      createNew: true,
+      updateExisting: true,
+      deleteMissing: true,
+      matchColumns: ['role_id', 'id_original', 'import_source_id'],
+      jsonArray: importEntities.role_databases,
+      deleteRestrictionColumns: ['import_source_id'],
+    },
+    {
+      name: 'role_tables',
+      findExisting: true,
+      createNew: true,
+      updateExisting: true,
+      deleteMissing: true,
+      matchColumns: ['role_id', 'id_original', 'import_source_id'],
+      jsonArray: importEntities.role_tables,
+      deleteRestrictionColumns: ['import_source_id'],
+    },
+  ];
+}
+
+module.exports = {
+  extractConnectionsFromEnv,
+  extractImportEntitiesFromEnv,
+  createStorageFromEnvReplicatorItems,
+};

packages/api/src/utility/getDiagramExport.js

@@ -1,21 +1,29 @@
-const getDiagramExport = (html, css, themeType, themeClassName, watermark) => {
+const getDiagramExport = (html, css, themeType, themeVariables, watermark) => {
   const watermarkHtml = watermark
     ? `
-        <div style="position: fixed; bottom: 0; right: 0; padding: 5px; font-size: 12px; color: var(--theme-font-2); background-color: var(--theme-bg-2); border-top-left-radius: 5px; border: 1px solid var(--theme-border);">
+        <div style="position: fixed; bottom: 0; right: 0; padding: 5px; font-size: 12px; color: var(--theme-generic-font-grayed); background-color: var(--theme-datagrid-background); border-top-left-radius: 5px; border: var(--theme-card-border);">
         ${watermark}
       </div>
   `
     : '';
+  
+  // Convert theme variables object to CSS custom properties
+  const themeVariablesCSS = themeVariables 
+    ? `:root {\n${Object.entries(themeVariables).map(([key, value]) => `  ${key}: ${value};`).join('\n')}\n}`
+    : '';
+  
   return `<html>
   <meta charset='utf-8'>
   
   <head>
       <style>
+        ${themeVariablesCSS}
+        
         ${css}
 
         body {
-          background: var(--theme-bg-1);
-          color: var(--theme-font-1);
+          background: var(--theme-datagrid-background);
+          color: var(--theme-generic-font);
         }
       </style>
 
@@ -55,7 +63,7 @@ const getDiagramExport = (html, css, themeType, themeClassName, watermark) => {
       </script>
   </head>
   
-  <body class='${themeType == 'dark' ? 'theme-type-dark' : 'theme-type-light'} ${themeClassName}' style='user-select:none; cursor:pointer'>
+  <body class='${themeType == 'dark' ? 'theme-type-dark' : 'theme-type-light'}' style='user-select:none; cursor:pointer'>
       ${html}
       ${watermarkHtml}
   </body>

packages/datalib/package.json

@@ -1,5 +1,5 @@
 {
-  "version": "6.0.0-alpha.1",
+  "version": "7.0.0-alpha.1",
   "name": "dbgate-datalib",
   "main": "lib/index.js",
   "typings": "lib/index.d.ts",
@@ -19,14 +19,14 @@
   ],
   "dependencies": {
     "date-fns": "^4.1.0",
-    "dbgate-filterparser": "^6.0.0-alpha.1",
-    "dbgate-sqltree": "^6.0.0-alpha.1",
-    "dbgate-tools": "^6.0.0-alpha.1",
+    "dbgate-filterparser": "^7.0.0-alpha.1",
+    "dbgate-sqltree": "^7.0.0-alpha.1",
+    "dbgate-tools": "^7.0.0-alpha.1",
     "uuid": "^3.4.0"
   },
   "devDependencies": {
     "@types/node": "^13.7.0",
-    "dbgate-types": "^6.0.0-alpha.1",
+    "dbgate-types": "^7.0.0-alpha.1",
     "jest": "^28.1.3",
     "ts-jest": "^28.0.7",
     "typescript": "^4.4.3"

packages/datalib/src/ChangeSetRedis.ts

@@ -0,0 +1,210 @@
+import { DatabaseMethodCallItem, DatabaseMethodCallList } from 'dbgate-types';
+
+export interface ChangeSetRedis_String {
+  key: string;
+  type: 'string';
+  value: string;
+}
+
+export interface ChangeSetRedis_JSON {
+  key: string;
+  type: 'json';
+  value: string;
+}
+
+export interface ChangeSetRedis_Hash {
+  key: string;
+  type: 'hash';
+  inserts: { key: string; value: string; ttl: number; editorRowId: string }[];
+  updates: { key: string; value: string; ttl: number }[];
+  deletes: string[];
+}
+
+export interface ChangeSetRedis_List {
+  key: string;
+  type: 'list';
+  inserts: { value: string; editorRowId: string }[];
+  updates: { index: number; value: string }[];
+  deletes: number[];
+}
+
+export interface ChangeSetRedis_Set {
+  key: string;
+  type: 'set';
+  inserts: { value: string; editorRowId: string }[];
+  deletes: string[];
+}
+
+export interface ChangeSetRedis_ZSet {
+  key: string;
+  type: 'zset';
+  inserts: { member: string; score: number; editorRowId: string }[];
+  updates: { member: string; score: number }[];
+  deletes: string[];
+}
+
+export interface ChangeSetRedis_Stream {
+  key: string;
+  type: 'stream';
+  generatedId?: string;
+  inserts: { field: string; value: string; editorRowId: string }[];
+  deletes: string[];
+}
+
+export type ChangeSetRedisType =
+  | ChangeSetRedis_String
+  | ChangeSetRedis_JSON
+  | ChangeSetRedis_Hash
+  | ChangeSetRedis_List
+  | ChangeSetRedis_Set
+  | ChangeSetRedis_ZSet
+  | ChangeSetRedis_Stream;
+
+export interface ChangeSetRedis {
+  changes: ChangeSetRedisType[];
+}
+
+export function redisChangeSetToRedisCommands(changeSet: ChangeSetRedis): DatabaseMethodCallList {
+  const calls: DatabaseMethodCallItem[] = [];
+
+  for (const change of changeSet.changes) {
+    if (change.type === 'string') {
+      calls.push({
+        method: 'SET',
+        args: [change.key, change.value],
+      });
+    } else if (change.type === 'json') {
+      calls.push({
+        method: 'JSON.SET',
+        args: [change.key, '$', change.value],
+      });
+    } else if (change.type === 'hash') {
+      if (change.updates && Array.isArray(change.updates)) {
+        for (const update of change.updates) {
+          calls.push({
+            method: 'HSET',
+            args: [change.key, update.key, update.value],
+          });
+
+          if (update.ttl !== undefined && update.ttl !== null && update.ttl !== -1) {
+            calls.push({
+              method: 'HEXPIRE',
+              args: [change.key, update.ttl, 'FIELDS', 1, update.key],
+            });
+          }
+        }
+      }
+
+      if (change.inserts && Array.isArray(change.inserts)) {
+        for (const insert of change.inserts) {
+          calls.push({
+            method: 'HSET',
+            args: [change.key, insert.key, insert.value],
+          });
+
+          if (insert.ttl !== undefined && insert.ttl !== null && insert.ttl !== -1) {
+            calls.push({
+              method: 'HEXPIRE',
+              args: [change.key, insert.ttl, 'FIELDS', 1, insert.key],
+            });
+          }
+        }
+      }
+
+      if (change.deletes && Array.isArray(change.deletes)) {
+        for (const delKey of change.deletes) {
+          calls.push({
+            method: 'HDEL',
+            args: [change.key, delKey],
+          });
+        }
+      }
+    } else if (change.type === 'zset') {
+      if (change.updates && Array.isArray(change.updates)) {
+        for (const update of change.updates) {
+          calls.push({
+            method: 'ZADD',
+            args: [change.key, update.score, update.member],
+          });
+        }
+      }
+
+      if (change.inserts && Array.isArray(change.inserts)) {
+        for (const insert of change.inserts) {
+          calls.push({
+            method: 'ZADD',
+            args: [change.key, insert.score, insert.member],
+          });
+        }
+      }
+
+      if (change.deletes && Array.isArray(change.deletes)) {
+        for (const delMember of change.deletes) {
+          calls.push({
+            method: 'ZREM',
+            args: [change.key, delMember],
+          });
+        }
+      }
+    } else if (change.type === 'list') {
+      if (change.updates && Array.isArray(change.updates)) {
+        for (const update of change.updates) {
+          calls.push({
+            method: 'LSET',
+            args: [change.key, update.index, update.value],
+          });
+        }
+      }
+
+      if (change.inserts && Array.isArray(change.inserts)) {
+        for (const insert of change.inserts) {
+          calls.push({
+            method: 'RPUSH',
+            args: [change.key, insert.value],
+          });
+        }
+      }
+    } else if (change.type === 'set') {
+      if (change.inserts && Array.isArray(change.inserts)) {
+        for (const insert of change.inserts) {
+          calls.push({
+            method: 'SADD',
+            args: [change.key, insert.value],
+          });
+        }
+      }
+
+      if (change.deletes && Array.isArray(change.deletes)) {
+        for (const delValue of change.deletes) {
+          calls.push({
+            method: 'SREM',
+            args: [change.key, delValue],
+          });
+        }
+      }
+    } else if (change.type === 'stream') {
+      if (change.inserts.length > 0) {
+        calls.push({
+          method: 'XADD',
+          args: [change.key, change.generatedId || '*', ...change.inserts.flatMap(f => [f.field, f.value])],
+        });
+      }
+      for (const delValue of change.deletes) {
+        calls.push({
+          method: 'XDEL',
+          args: [change.key, delValue],
+        });
+      }
+    }
+  }
+
+  return { calls };
+}
+
+export function convertRedisCallListToScript(callList: DatabaseMethodCallList): string {
+  let script = '';
+  for (const call of callList.calls) {
+    script += `${call.method} ${call.args.map(arg => (typeof arg === 'string' ? `"${arg}"` : arg)).join(' ')}\n`;
+  }
+  return script;
+}

packages/datalib/src/DataReplicator.ts

@@ -23,6 +23,7 @@ export interface DataReplicatorItem {
   deleteMissing: boolean;
   deleteRestrictionColumns: string[];
   matchColumns: string[];
+  skipUpdateColumns?: string[];
 }
 
 export interface DataReplicatorOptions {
@@ -151,7 +152,12 @@ class ReplicatorItemHolder {
         chunk,
         this.table.columns.map(x => x.columnName)
       ),
-      [this.autoColumn, ...this.backReferences.map(x => x.columnName), ...this.references.map(x => x.columnName)]
+      [
+        this.autoColumn,
+        ...this.backReferences.map(x => x.columnName),
+        ...this.references.map(x => x.columnName),
+        ...(this.item.skipUpdateColumns || []),
+      ]
     );
 
     return res;

packages/datalib/src/GridDisplay.ts

@@ -451,7 +451,7 @@ export abstract class GridDisplay {
       ...cfg,
       filters: _.omit(cfg.filters, [uniqueName]),
       formFilterColumns: (cfg.formFilterColumns || []).filter(x => x != uniqueName),
-      disabledFilterColumns: (cfg.disabledFilterColumns).filter(x => x != uniqueName),
+      disabledFilterColumns: cfg.disabledFilterColumns.filter(x => x != uniqueName),
     }));
     this.reload();
   }
@@ -541,6 +541,7 @@ export abstract class GridDisplay {
       const column = (this.baseTable || this.baseView)?.columns?.find(x => x.columnName == uniqueName);
       if (isTypeLogical(column?.dataType)) return 'COUNT DISTINCT';
       if (column?.autoIncrement) return 'COUNT';
+      if (this.driver?.dialect?.disableGroupingForDataType?.(column?.dataType)) return 'NULL';
       return 'MAX';
     }
     return null;

packages/datalib/src/index.ts

@@ -25,3 +25,4 @@ export * from './CustomGridDisplay';
 export * from './ScriptDrivedDeployer';
 export * from './chartDefinitions';
 export * from './chartProcessor';
+export * from './ChangeSetRedis';

packages/dbmodel/package.json

@@ -1,6 +1,6 @@
 {
   "name": "dbmodel",
-  "version": "6.0.0-alpha.1",
+  "version": "7.0.0-alpha.1",
   "homepage": "https://dbgate.org/",
   "repository": {
     "type": "git",
@@ -30,16 +30,16 @@
   ],
   "dependencies": {
     "commander": "^10.0.0",
-    "dbgate-api": "^6.0.0-alpha.1",
-    "dbgate-plugin-csv": "^6.0.0-alpha.1",
-    "dbgate-plugin-excel": "^6.0.0-alpha.1",
-    "dbgate-plugin-mongo": "^6.0.0-alpha.1",
-    "dbgate-plugin-mssql": "^6.0.0-alpha.1",
-    "dbgate-plugin-mysql": "^6.0.0-alpha.1",
-    "dbgate-plugin-postgres": "^6.0.0-alpha.1",
-    "dbgate-plugin-xml": "^6.0.0-alpha.1",
-    "dbgate-plugin-oracle": "^6.0.0-alpha.1",
-    "dbgate-web": "^6.0.0-alpha.1",
+    "dbgate-api": "^7.0.0-alpha.1",
+    "dbgate-plugin-csv": "^7.0.0-alpha.1",
+    "dbgate-plugin-excel": "^7.0.0-alpha.1",
+    "dbgate-plugin-mongo": "^7.0.0-alpha.1",
+    "dbgate-plugin-mssql": "^7.0.0-alpha.1",
+    "dbgate-plugin-mysql": "^7.0.0-alpha.1",
+    "dbgate-plugin-postgres": "^7.0.0-alpha.1",
+    "dbgate-plugin-xml": "^7.0.0-alpha.1",
+    "dbgate-plugin-oracle": "^7.0.0-alpha.1",
+    "dbgate-web": "^7.0.0-alpha.1",
     "dotenv": "^16.0.0",
     "pinomin": "^1.0.5"
   }

packages/filterparser/package.json

@@ -1,5 +1,5 @@
 {
-  "version": "6.0.0-alpha.1",
+  "version": "7.0.0-alpha.1",
   "name": "dbgate-filterparser",
   "main": "lib/index.js",
   "typings": "lib/index.d.ts",
@@ -17,7 +17,7 @@
     "lib"
   ],
   "devDependencies": {
-    "dbgate-types": "^6.0.0-alpha.1",
+    "dbgate-types": "^7.0.0-alpha.1",
     "@types/jest": "^25.1.4",
     "@types/node": "^13.7.0",
     "jest": "^28.1.3",
@@ -26,7 +26,7 @@
   },
   "dependencies": {
     "@types/parsimmon": "^1.10.1",
-    "dbgate-tools": "^6.0.0-alpha.1",
+    "dbgate-tools": "^7.0.0-alpha.1",
     "lodash": "^4.17.21",
     "date-fns": "^4.1.0",
     "moment": "^2.24.0",

packages/filterparser/src/filterTool.ts

@@ -21,6 +21,7 @@ export function getFilterValueExpression(value, dataType?) {
   if (value === false) return 'FALSE';
   if (value.$oid) return `ObjectId("${value.$oid}")`;
   if (value.$bigint) return value.$bigint;
+  if (value.$decimal) return value.$decimal;
   if (value.type == 'Buffer' && Array.isArray(value.data)) {
     return '0x' + arrayToHexString(value.data);
   }

packages/serve/package.json

@@ -1,6 +1,6 @@
 {
   "name": "dbgate-serve",
-  "version": "6.0.0-alpha.1",
+  "version": "7.0.0-alpha.1",
   "homepage": "https://dbgate.org/",
   "repository": {
     "type": "git",
@@ -18,19 +18,19 @@
     "web"
   ],
   "dependencies": {
-    "dbgate-api": "^6.0.0-alpha.1",
-    "dbgate-plugin-clickhouse": "^6.0.0-alpha.1",
-    "dbgate-plugin-csv": "^6.0.0-alpha.1",
-    "dbgate-plugin-excel": "^6.0.0-alpha.1",
-    "dbgate-plugin-mongo": "^6.0.0-alpha.1",
-    "dbgate-plugin-mssql": "^6.0.0-alpha.1",
-    "dbgate-plugin-mysql": "^6.0.0-alpha.1",
-    "dbgate-plugin-oracle": "^6.0.0-alpha.1",
-    "dbgate-plugin-postgres": "^6.0.0-alpha.1",
-    "dbgate-plugin-redis": "^6.0.0-alpha.1",
-    "dbgate-plugin-sqlite": "^6.0.0-alpha.1",
-    "dbgate-plugin-xml": "^6.0.0-alpha.1",
-    "dbgate-web": "^6.0.0-alpha.1",
+    "dbgate-api": "^7.0.0-alpha.1",
+    "dbgate-plugin-clickhouse": "^7.0.0-alpha.1",
+    "dbgate-plugin-csv": "^7.0.0-alpha.1",
+    "dbgate-plugin-excel": "^7.0.0-alpha.1",
+    "dbgate-plugin-mongo": "^7.0.0-alpha.1",
+    "dbgate-plugin-mssql": "^7.0.0-alpha.1",
+    "dbgate-plugin-mysql": "^7.0.0-alpha.1",
+    "dbgate-plugin-oracle": "^7.0.0-alpha.1",
+    "dbgate-plugin-postgres": "^7.0.0-alpha.1",
+    "dbgate-plugin-redis": "^7.0.0-alpha.1",
+    "dbgate-plugin-sqlite": "^7.0.0-alpha.1",
+    "dbgate-plugin-xml": "^7.0.0-alpha.1",
+    "dbgate-web": "^7.0.0-alpha.1",
     "dotenv": "^16.0.0"
   }
 }

packages/sqltree/package.json

@@ -1,5 +1,5 @@
 {
-  "version": "6.0.0-alpha.1",
+  "version": "7.0.0-alpha.1",
   "name": "dbgate-sqltree",
   "main": "lib/index.js",
   "typings": "lib/index.d.ts",
@@ -27,7 +27,7 @@
   ],
   "devDependencies": {
     "@types/node": "^13.7.0",
-    "dbgate-types": "^6.0.0-alpha.1",
+    "dbgate-types": "^7.0.0-alpha.1",
     "typescript": "^4.4.3"
   },
   "dependencies": {

packages/sqltree/src/dumpSqlCondition.ts

@@ -19,14 +19,28 @@ export function dumpSqlCondition(dmp: SqlDumper, condition: Condition) {
       dmp.put(' ^is ^not ^null');
       break;
     case 'isEmpty':
-      dmp.put('^trim(');
-      dumpSqlExpression(dmp, condition.expr);
-      dmp.put(") = ''");
+      // Use DATALENGTH for MSSQL TEXT/NTEXT/IMAGE columns to avoid TRIM error
+      if (dmp.dialect.useDatalengthForEmptyString?.(condition.expr?.['dataType'])) {
+        dmp.put('^datalength(');
+        dumpSqlExpression(dmp, condition.expr);
+        dmp.put(') = 0');
+      } else {
+        dmp.put('^trim(');
+        dumpSqlExpression(dmp, condition.expr);
+        dmp.put(") = ''");
+      }
       break;
     case 'isNotEmpty':
-      dmp.put('^trim(');
-      dumpSqlExpression(dmp, condition.expr);
-      dmp.put(") <> ''");
+      // Use DATALENGTH for MSSQL TEXT/NTEXT/IMAGE columns to avoid TRIM error
+      if (dmp.dialect.useDatalengthForEmptyString?.(condition.expr?.['dataType'])) {
+        dmp.put('^datalength(');
+        dumpSqlExpression(dmp, condition.expr);
+        dmp.put(') > 0');
+      } else {
+        dmp.put('^trim(');
+        dumpSqlExpression(dmp, condition.expr);
+        dmp.put(") <> ''");
+      }
       break;
     case 'and':
     case 'or':

packages/sqltree/src/evaluateCondition.ts

@@ -19,6 +19,7 @@ function isLike(value, test) {
 function extractRawValue(value) {
   if (value?.$bigint) return value.$bigint;
   if (value?.$oid) return value.$oid;
+  if (value?.$decimal) return value.$decimal;
   return value;
 }
 

packages/tools/package.json

@@ -1,5 +1,5 @@
 {
-  "version": "6.0.0-alpha.1",
+  "version": "7.0.0-alpha.1",
   "name": "dbgate-tools",
   "main": "lib/index.js",
   "typings": "lib/index.d.ts",
@@ -26,7 +26,7 @@
   ],
   "devDependencies": {
     "@types/node": "^13.7.0",
-    "dbgate-types": "^6.0.0-alpha.1",
+    "dbgate-types": "^7.0.0-alpha.1",
     "jest": "^28.1.3",
     "ts-jest": "^28.0.7",
     "typescript": "^4.4.3"
@@ -34,7 +34,7 @@
   "dependencies": {
     "blueimp-md5": "^2.19.0",
     "dbgate-query-splitter": "^4.11.9",
-    "dbgate-sqltree": "^6.0.0-alpha.1",
+    "dbgate-sqltree": "^7.0.0-alpha.1",
     "debug": "^4.3.4",
     "json-stable-stringify": "^1.0.1",
     "lodash": "^4.17.21",

packages/tools/src/DatabaseAnalyser.ts

@@ -164,6 +164,11 @@ export class DatabaseAnalyser<TClient = any> {
 
     const res = {};
     for (const field of STRUCTURE_FIELDS) {
+      const isAll = this.modifications.some(x => x.action == 'all' && x.objectTypeField == field);
+      if (isAll) {
+        res[field] = newlyAnalysed[field] || [];
+        continue;
+      }
       const removedIds = this.modifications
         .filter(x => x.action == 'remove' && x.objectTypeField == field)
         .map(x => x.objectId);

packages/tools/src/SqlDumper.ts

@@ -26,6 +26,7 @@ import _isDate from 'lodash/isDate';
 import _isArray from 'lodash/isArray';
 import _isPlainObject from 'lodash/isPlainObject';
 import _keys from 'lodash/keys';
+import _cloneDeep from 'lodash/cloneDeep';
 import uuidv1 from 'uuid/v1';
 
 export class SqlDumper implements AlterProcessor {
@@ -87,6 +88,7 @@ export class SqlDumper implements AlterProcessor {
       this.putByteArrayValue(bytes);
     }
     else if (value?.$bigint) this.putRaw(value?.$bigint);
+    else if (value?.$decimal) this.putRaw(value?.$decimal);
     else if (_isPlainObject(value) || _isArray(value)) this.putStringValue(JSON.stringify(value));
     else this.put('^null');
   }
@@ -542,9 +544,14 @@ export class SqlDumper implements AlterProcessor {
     }
     this.endCommand();
   }
+  indexType(ix: IndexInfo) {
+    if (ix.isUnique) {
+      this.put(' ^unique');
+    }
+  }
   createIndex(ix: IndexInfo) {
     this.put('^create');
-    if (ix.isUnique) this.put(' ^unique');
+    this.indexType(ix);
     this.put(' ^index %i &n^on %f (&>&n', ix.constraintName, ix);
     this.putCollection(',&n', ix.columns, col => {
       this.put('%i %k', col.columnName, col.isDescending == true ? 'DESC' : 'ASC');
@@ -666,6 +673,68 @@ export class SqlDumper implements AlterProcessor {
     }
   }
 
+  sanitizeTableConstraints(table: TableInfo): TableInfo {
+    // Create a deep copy of the table
+    const sanitized = _cloneDeep(table);
+    
+    // Get the set of existing column names
+    const existingColumns = new Set(sanitized.columns.map(col => col.columnName));
+    
+    // Filter primary key columns to only include existing columns
+    if (sanitized.primaryKey) {
+      const validPkColumns = sanitized.primaryKey.columns.filter(col => existingColumns.has(col.columnName));
+      if (validPkColumns.length === 0) {
+        // If no valid columns remain, remove the primary key entirely
+        sanitized.primaryKey = null;
+      } else if (validPkColumns.length < sanitized.primaryKey.columns.length) {
+        // Update primary key with only valid columns
+        sanitized.primaryKey = {
+          ...sanitized.primaryKey,
+          columns: validPkColumns
+        };
+      }
+    }
+    
+    // Filter sorting key columns to only include existing columns
+    if (sanitized.sortingKey) {
+      const validSkColumns = sanitized.sortingKey.columns.filter(col => existingColumns.has(col.columnName));
+      if (validSkColumns.length === 0) {
+        sanitized.sortingKey = null;
+      } else if (validSkColumns.length < sanitized.sortingKey.columns.length) {
+        sanitized.sortingKey = {
+          ...sanitized.sortingKey,
+          columns: validSkColumns
+        };
+      }
+    }
+    
+    // Filter foreign keys to only include those with all columns present
+    if (sanitized.foreignKeys) {
+      sanitized.foreignKeys = sanitized.foreignKeys.filter(fk =>
+        fk.columns.every(col => existingColumns.has(col.columnName))
+      );
+    }
+    
+    // Filter indexes to only include those with all columns present
+    if (sanitized.indexes) {
+      sanitized.indexes = sanitized.indexes.filter(idx =>
+        idx.columns.every(col => existingColumns.has(col.columnName))
+      );
+    }
+    
+    // Filter unique constraints to only include those with all columns present
+    if (sanitized.uniques) {
+      sanitized.uniques = sanitized.uniques.filter(uq =>
+        uq.columns.every(col => existingColumns.has(col.columnName))
+      );
+    }
+    
+    // Filter dependencies (references from other tables) - these should remain as-is
+    // since they don't affect the CREATE TABLE statement for this table
+    
+    return sanitized;
+  }
+
   recreateTable(oldTable: TableInfo, newTable: TableInfo) {
     if (!oldTable.pairingId || !newTable.pairingId || oldTable.pairingId != newTable.pairingId) {
       throw new Error('Recreate is not possible: oldTable.paringId != newTable.paringId');
@@ -680,48 +749,51 @@ export class SqlDumper implements AlterProcessor {
       }))
       .filter(x => x.newcol);
 
+    // Create a sanitized version of newTable with constraints that only reference existing columns
+    const sanitizedNewTable = this.sanitizeTableConstraints(newTable);
+
     if (this.driver.supportsTransactions) {
       this.dropConstraints(oldTable, true);
       this.renameTable(oldTable, tmpTable);
 
-      this.createTable(newTable);
+      this.createTable(sanitizedNewTable);
 
-      const autoinc = newTable.columns.find(x => x.autoIncrement);
+      const autoinc = sanitizedNewTable.columns.find(x => x.autoIncrement);
       if (autoinc) {
-        this.allowIdentityInsert(newTable, true);
+        this.allowIdentityInsert(sanitizedNewTable, true);
       }
 
       this.putCmd(
         '^insert ^into %f (%,i) select %,i ^from %f',
-        newTable,
+        sanitizedNewTable,
         columnPairs.map(x => x.newcol.columnName),
         columnPairs.map(x => x.oldcol.columnName),
         { ...oldTable, pureName: tmpTable }
       );
 
       if (autoinc) {
-        this.allowIdentityInsert(newTable, false);
+        this.allowIdentityInsert(sanitizedNewTable, false);
       }
 
       if (this.dialect.dropForeignKey) {
-        newTable.dependencies.forEach(cnt => this.createConstraint(cnt));
+        sanitizedNewTable.dependencies.forEach(cnt => this.createConstraint(cnt));
       }
 
       this.dropTable({ ...oldTable, pureName: tmpTable });
     } else {
       // we have to preserve old table as long as possible
-      this.createTable({ ...newTable, pureName: tmpTable });
+      this.createTable({ ...sanitizedNewTable, pureName: tmpTable });
 
       this.putCmd(
         '^insert ^into %f (%,i) select %,s ^from %f',
-        { ...newTable, pureName: tmpTable },
+        { ...sanitizedNewTable, pureName: tmpTable },
         columnPairs.map(x => x.newcol.columnName),
         columnPairs.map(x => x.oldcol.columnName),
         oldTable
       );
 
       this.dropTable(oldTable);
-      this.renameTable({ ...newTable, pureName: tmpTable }, newTable.pureName);
+      this.renameTable({ ...sanitizedNewTable, pureName: tmpTable }, newTable.pureName);
     }
   }
 

packages/tools/src/SqlGenerator.ts

@@ -27,6 +27,7 @@ interface SqlGeneratorOptions {
   createIndexes: boolean;
   insert: boolean;
   skipAutoincrementColumn: boolean;
+  skipComputedColumns: boolean;
   disableConstraints: boolean;
   omitNulls: boolean;
   truncate: boolean;
@@ -260,9 +261,12 @@ export class SqlGenerator {
   }
 
   processReadable(table: TableInfo, readable) {
-    const columnsFiltered = this.options.skipAutoincrementColumn
+    const columnsFilteredPre = this.options.skipAutoincrementColumn
       ? table.columns.filter(x => !x.autoIncrement)
       : table.columns;
+    const columnsFiltered = this.options.skipComputedColumns
+      ? columnsFilteredPre.filter(x => !x.computedExpression)
+      : columnsFilteredPre;
     const columnNames = columnsFiltered.map(x => x.columnName);
     let isClosed = false;
     let isHeaderRead = false;

packages/tools/src/alterPlan.ts

@@ -91,8 +91,8 @@ interface AlterOperation_RenameConstraint {
 }
 interface AlterOperation_RecreateTable {
   operationType: 'recreateTable';
-  table: TableInfo;
-  operations: AlterOperation[];
+  oldTable: TableInfo;
+  newTable: TableInfo;
 }
 interface AlterOperation_FillPreloadedRows {
   operationType: 'fillPreloadedRows';
@@ -249,11 +249,11 @@ export class AlterPlan {
     });
   }
 
-  recreateTable(table: TableInfo, operations: AlterOperation[]) {
+  recreateTable(oldTable: TableInfo, newTable: TableInfo) {
     this.operations.push({
       operationType: 'recreateTable',
-      table,
-      operations,
+      oldTable,
+      newTable,
     });
     this.recreates.tables += 1;
   }
@@ -337,7 +337,13 @@ export class AlterPlan {
             return opRes;
           }),
           op,
-        ];
+        ].filter(op => {
+          // filter duplicated drops
+          const existingDrop = this.operations.find(
+            o => o.operationType == 'dropConstraint' && o.oldObject === op['oldObject']
+          );
+          return existingDrop == null;
+        });
         return res;
       }
 
@@ -498,53 +504,121 @@ export class AlterPlan {
         return [];
       }
 
-      const table = this.wholeNewDb.tables.find(
+      const oldTable = this.wholeOldDb.tables.find(
+        x => x.pureName == op[objectField].pureName && x.schemaName == op[objectField].schemaName
+      );
+      const newTable = this.wholeNewDb.tables.find(
         x => x.pureName == op[objectField].pureName && x.schemaName == op[objectField].schemaName
       );
       this.recreates.tables += 1;
       return [
         {
           operationType: 'recreateTable',
-          table,
-          operations: [op],
+          oldTable,
+          newTable,
+          // operations: [op],
         },
       ];
     }
     return null;
   }
 
-  _groupTableRecreations(): AlterOperation[] {
-    const res = [];
-    const recreates = {};
+  _removeRecreatedTableAlters(): AlterOperation[] {
+    const res: AlterOperation[] = [];
+    const recreates = new Set<string>();
     for (const op of this.operations) {
-      if (op.operationType == 'recreateTable' && op.table) {
-        const existingRecreate = recreates[`${op.table.schemaName}||${op.table.pureName}`];
-        if (existingRecreate) {
-          existingRecreate.operations.push(...op.operations);
-        } else {
-          const recreate = {
-            ...op,
-            operations: [...op.operations],
-          };
-          res.push(recreate);
-          recreates[`${op.table.schemaName}||${op.table.pureName}`] = recreate;
-        }
-      } else {
-        // @ts-ignore
-        const oldObject: TableInfo = op.oldObject || op.object;
-        if (oldObject) {
-          const recreated = recreates[`${oldObject.schemaName}||${oldObject.pureName}`];
-          if (recreated) {
-            recreated.operations.push(op);
-            continue;
-          }
-        }
-        res.push(op);
+      if (op.operationType == 'recreateTable' && op.oldTable && op.newTable) {
+        const key = `${op.oldTable.schemaName}||${op.oldTable.pureName}`;
+        recreates.add(key);
       }
     }
+
+    for (const op of this.operations) {
+      switch (op.operationType) {
+        case 'createColumn':
+        case 'createConstraint':
+          {
+            const key = `${op.newObject.schemaName}||${op.newObject.pureName}`;
+            if (recreates.has(key)) {
+              // skip create inside recreated table
+              continue;
+            }
+          }
+          break;
+        case 'dropColumn':
+        case 'dropConstraint':
+        case 'changeColumn':
+          {
+            const key = `${op.oldObject.schemaName}||${op.oldObject.pureName}`;
+            if (recreates.has(key)) {
+              // skip drop/change inside recreated table
+              continue;
+            }
+          }
+          break;
+        case 'renameColumn':
+          {
+            const key = `${op.object.schemaName}||${op.object.pureName}`;
+            if (recreates.has(key)) {
+              // skip rename inside recreated table
+              continue;
+            }
+          }
+          break;
+      }
+      res.push(op);
+    }
     return res;
   }
 
+  _groupTableRecreations(): AlterOperation[] {
+    const res = [];
+    const recreates = new Set<string>();
+    for (const op of this.operations) {
+      if (op.operationType == 'recreateTable' && op.oldTable && op.newTable) {
+        const key = `${op.oldTable.schemaName}||${op.oldTable.pureName}`;
+        if (recreates.has(key)) {
+          // prevent duplicate recreates
+          continue;
+        }
+        recreates.add(key);
+      }
+
+      res.push(op);
+    }
+    return res;
+
+    // const res = [];
+    // const recreates = {};
+    // for (const op of this.operations) {
+    //   if (op.operationType == 'recreateTable' && op.table) {
+    //     const existingRecreate = recreates[`${op.table.schemaName}||${op.table.pureName}`];
+    //     if (existingRecreate) {
+    //       existingRecreate.operations.push(...op.operations);
+    //     } else {
+    //       const recreate = {
+    //         ...op,
+    //         operations: [...op.operations],
+    //       };
+    //       res.push(recreate);
+    //       recreates[`${op.table.schemaName}||${op.table.pureName}`] = recreate;
+    //     }
+    //   } else {
+    //     // @ts-ignore
+    //     const oldObject: TableInfo = op.oldObject || op.object;
+    //     if (oldObject) {
+    //       const recreated = recreates[`${oldObject.schemaName}||${oldObject.pureName}`];
+    //       if (recreated) {
+    //         recreated.operations.push(op);
+    //         continue;
+    //       }
+    //     }
+    //     res.push(op);
+    //   }
+    // }
+    // return res;
+  }
+
   _moveForeignKeysToLast(): AlterOperation[] {
     if (!this.dialect.createForeignKey) {
       return this.operations;
@@ -611,6 +685,8 @@ export class AlterPlan {
 
     // console.log('*****************OPERATIONS3', this.operations);
 
+    this.operations = this._removeRecreatedTableAlters();
+
     this.operations = this._moveForeignKeysToLast();
 
     // console.log('*****************OPERATIONS4', this.operations);
@@ -673,16 +749,16 @@ export function runAlterOperation(op: AlterOperation, processor: AlterProcessor)
       break;
     case 'recreateTable':
       {
-        const oldTable = generateTablePairingId(op.table);
-        const newTable = _.cloneDeep(oldTable);
-        const newDb = DatabaseAnalyser.createEmptyStructure();
-        newDb.tables.push(newTable);
-        // console.log('////////////////////////////newTable1', newTable);
-        op.operations.forEach(child => runAlterOperation(child, new DatabaseInfoAlterProcessor(newDb)));
-        // console.log('////////////////////////////op.operations', op.operations);
-        // console.log('////////////////////////////op.table', op.table);
-        // console.log('////////////////////////////newTable2', newTable);
-        processor.recreateTable(oldTable, newTable);
+        // const oldTable = generateTablePairingId(op.table);
+        // const newTable = _.cloneDeep(oldTable);
+        // const newDb = DatabaseAnalyser.createEmptyStructure();
+        // newDb.tables.push(newTable);
+        // // console.log('////////////////////////////newTable1', newTable);
+        // op.operations.forEach(child => runAlterOperation(child, new DatabaseInfoAlterProcessor(newDb)));
+        // // console.log('////////////////////////////op.operations', op.operations);
+        // // console.log('////////////////////////////op.table', op.table);
+        // // console.log('////////////////////////////newTable2', newTable);
+        processor.recreateTable(op.oldTable, op.newTable);
       }
       break;
   }

packages/tools/src/dbKeysLoader.ts

@@ -1,351 +0,0 @@
-import _omit from 'lodash/omit';
-import _sortBy from 'lodash/sortBy';
-
-export const DB_KEYS_SHOW_INCREMENT = 100;
-
-export interface DbKeysNodeModelBase {
-  text?: string;
-  sortKey: string;
-  key: string;
-  count?: number;
-  level: number;
-  keyPath: string[];
-  parentKey: string;
-}
-
-export interface DbKeysLeafNodeModel extends DbKeysNodeModelBase {
-  type: 'string' | 'hash' | 'set' | 'list' | 'zset' | 'stream' | 'binary' | 'ReJSON-RL';
-}
-
-export interface DbKeysFolderNodeModel extends DbKeysNodeModelBase {
-  // root: string;
-  type: 'dir';
-  // visibleCount?: number;
-  // isExpanded?: boolean;
-}
-
-export interface DbKeysFolderStateMode {
-  key: string;
-  visibleCount?: number;
-  isExpanded?: boolean;
-}
-
-export interface DbKeysTreeModel {
-  treeKeySeparator: string;
-  root: DbKeysFolderNodeModel;
-  dirsByKey: { [key: string]: DbKeysFolderNodeModel };
-  dirStateByKey: { [key: string]: DbKeysFolderStateMode };
-  childrenByKey: { [key: string]: DbKeysNodeModel[] };
-  keyObjectsByKey: { [key: string]: DbKeysNodeModel };
-  scannedKeys: number;
-  loadCount: number;
-  dbsize: number;
-  cursor: string;
-  loadedAll: boolean;
-  // refreshAll?: boolean;
-}
-
-export type DbKeysNodeModel = DbKeysLeafNodeModel | DbKeysFolderNodeModel;
-
-export interface DbKeyLoadedModel {
-  key: string;
-
-  type: 'string' | 'hash' | 'set' | 'list' | 'zset' | 'stream' | 'binary' | 'ReJSON-RL';
-  count?: number;
-}
-
-export interface DbKeysLoadResult {
-  nextCursor: string;
-  keys: DbKeyLoadedModel[];
-  dbsize: number;
-}
-
-// export type DbKeysLoadFunction = (root: string, limit: number) => Promise<DbKeysLoadResult>;
-
-export type DbKeysChangeModelFunction = (
-  func: (model: DbKeysTreeModel) => DbKeysTreeModel,
-  loadNextPage: boolean
-) => void;
-
-// function dbKeys_findFolderNode(node: DbKeysNodeModel, root: string) {
-//   if (node.type != 'dir') {
-//     return null;
-//   }
-//   if (node.root === root) {
-//     return node;
-//   }
-//   for (const child of node.children ?? []) {
-//     const res = dbKeys_findFolderNode(child, root);
-//     if (res) {
-//       return res;
-//     }
-//   }
-//   return null;
-// }
-
-// export async function dbKeys_loadKeysFromNode(
-//   tree: DbKeysTreeModel,
-//   callingRoot: string,
-//   separator: string,
-//   loader: DbKeysLoadFunction
-// ): Promise<DbKeysTreeModel> {
-//   const callingRootNode = tree.dirsByKey[callingRoot];
-//   if (!callingRootNode) {
-//     return tree;
-//   }
-//   const newItems = await loader(callingRoot, callingRootNode.maxShowCount ?? SHOW_INCREMENT);
-
-//   return {
-//     ...tree,
-//     childrenByKey: {
-//       ...tree.childrenByKey,
-//       [callingRoot]: newItems,
-//     },
-//   };
-// }
-
-// export async function dbKeys_loadMissing(tree: DbKeysTreeModel, loader: DbKeysLoadFunction): Promise<DbKeysTreeModel> {
-//   const childrenByKey = { ...tree.childrenByKey };
-//   const dirsByKey = { ...tree.dirsByKey };
-
-//   for (const root in tree.dirsByKey) {
-//     const dir = tree.dirsByKey[root];
-
-//     if (dir.isExpanded && dir.shouldLoadNext) {
-//       if (!tree.childrenByKey[root] || dir.hasNext) {
-//         const loadCount = dir.maxShowCount && dir.shouldLoadNext ? dir.maxShowCount + SHOW_INCREMENT : SHOW_INCREMENT;
-//         const items = await loader(root, loadCount + 1);
-
-//         childrenByKey[root] = items.slice(0, loadCount);
-//         dirsByKey[root] = {
-//           ...dir,
-//           shouldLoadNext: false,
-//           maxShowCount: loadCount,
-//           hasNext: items.length > loadCount,
-//         };
-
-//         for (const child of items.slice(0, loadCount)) {
-//           if (child.type == 'dir' && !dirsByKey[child.root]) {
-//             dirsByKey[child.root] = {
-//               shouldLoadNext: false,
-//               maxShowCount: null,
-//               hasNext: false,
-//               isExpanded: false,
-//               type: 'dir',
-//               level: dir.level + 1,
-//               root: child.root,
-//               text: child.text,
-//             };
-//           }
-//         }
-//       } else {
-//         dirsByKey[root] = {
-//           ...dir,
-//           shouldLoadNext: false,
-//         };
-//       }
-//     }
-//   }
-
-//   return {
-//     ...tree,
-//     dirsByKey,
-//     childrenByKey,
-//     refreshAll: false,
-//   };
-// }
-
-export function dbKeys_mergeNextPage(tree: DbKeysTreeModel, nextPage: DbKeysLoadResult): DbKeysTreeModel {
-  const keyObjectsByKey = { ...tree.keyObjectsByKey };
-
-  for (const keyObj of nextPage.keys) {
-    const keyPath = keyObj.key.split(tree.treeKeySeparator);
-    keyObjectsByKey[keyObj.key] = {
-      ...keyObj,
-      level: keyPath.length,
-      text: keyPath[keyPath.length - 1],
-      sortKey: keyPath[keyPath.length - 1],
-      keyPath,
-      parentKey: keyPath.slice(0, -1).join(tree.treeKeySeparator),
-    };
-  }
-
-  const dirsByKey: { [key: string]: DbKeysFolderNodeModel } = {};
-  const childrenByKey: { [key: string]: DbKeysNodeModel[] } = {};
-
-  dirsByKey[''] = tree.root;
-
-  for (const keyObj of Object.values(keyObjectsByKey)) {
-    const dirPath = keyObj.keyPath.slice(0, -1);
-    const dirKey = dirPath.join(tree.treeKeySeparator);
-
-    let dirDepth = keyObj.keyPath.length - 1;
-
-    while (dirDepth > 0) {
-      const newDirPath = keyObj.keyPath.slice(0, dirDepth);
-      const newDirKey = newDirPath.join(tree.treeKeySeparator);
-      if (!dirsByKey[newDirKey]) {
-        dirsByKey[newDirKey] = {
-          level: keyObj.level - 1,
-          keyPath: newDirPath,
-          parentKey: newDirPath.slice(0, -1).join(tree.treeKeySeparator),
-          type: 'dir',
-          key: newDirKey,
-          text: `${newDirPath[newDirPath.length - 1]}${tree.treeKeySeparator}*`,
-          sortKey: newDirPath[newDirPath.length - 1],
-        };
-      }
-
-      dirDepth -= 1;
-    }
-
-    if (!childrenByKey[dirKey]) {
-      childrenByKey[dirKey] = [];
-    }
-
-    childrenByKey[dirKey].push(keyObj);
-  }
-
-  for (const dirObj of Object.values(dirsByKey)) {
-    if (dirObj.key == '') {
-      continue;
-    }
-
-    if (!childrenByKey[dirObj.parentKey]) {
-      childrenByKey[dirObj.parentKey] = [];
-    }
-    childrenByKey[dirObj.parentKey].push(dirObj);
-
-    // set key count
-    dirsByKey[dirObj.key].count = childrenByKey[dirObj.key].length;
-  }
-
-  for (const key in childrenByKey) {
-    childrenByKey[key] = _sortBy(childrenByKey[key], 'sortKey');
-  }
-
-  return {
-    ...tree,
-    cursor: nextPage.nextCursor,
-    dirsByKey,
-    childrenByKey,
-    keyObjectsByKey,
-    scannedKeys: tree.scannedKeys + tree.loadCount,
-    loadedAll: nextPage.nextCursor == '0',
-    dbsize: nextPage.dbsize,
-  };
-}
-
-export function dbKeys_markNodeExpanded(tree: DbKeysTreeModel, root: string, isExpanded: boolean): DbKeysTreeModel {
-  const node = tree.dirStateByKey[root];
-  return {
-    ...tree,
-    dirStateByKey: {
-      ...tree.dirStateByKey,
-      [root]: {
-        ...node,
-        isExpanded,
-      },
-    },
-  };
-}
-
-export function dbKeys_showNextItems(tree: DbKeysTreeModel, root: string): DbKeysTreeModel {
-  const node = tree.dirStateByKey[root];
-  return {
-    ...tree,
-    dirStateByKey: {
-      ...tree.dirStateByKey,
-      [root]: {
-        ...node,
-        visibleCount: (node?.visibleCount ?? DB_KEYS_SHOW_INCREMENT) + DB_KEYS_SHOW_INCREMENT,
-      },
-    },
-  };
-}
-
-export function dbKeys_createNewModel(treeKeySeparator: string): DbKeysTreeModel {
-  const root: DbKeysFolderNodeModel = {
-    level: 0,
-    type: 'dir',
-    keyPath: [],
-    parentKey: '',
-    key: '',
-    sortKey: '',
-  };
-  return {
-    treeKeySeparator,
-    childrenByKey: {},
-    keyObjectsByKey: {},
-    dirsByKey: {
-      '': root,
-    },
-    dirStateByKey: {
-      '': {
-        key: '',
-        visibleCount: DB_KEYS_SHOW_INCREMENT,
-        isExpanded: true,
-      },
-    },
-    scannedKeys: 0,
-    dbsize: 0,
-    loadCount: 2000,
-    cursor: '0',
-    root,
-    loadedAll: false,
-  };
-}
-
-export function dbKeys_clearLoadedData(tree: DbKeysTreeModel): DbKeysTreeModel {
-  return {
-    ...tree,
-    childrenByKey: {},
-    keyObjectsByKey: {},
-    dirsByKey: {
-      '': tree.root,
-    },
-    scannedKeys: 0,
-    dbsize: 0,
-    cursor: '0',
-    loadedAll: false,
-  };
-}
-
-// export function dbKeys_reloadFolder(tree: DbKeysTreeModel, root: string): DbKeysTreeModel {
-//   return {
-//     ...tree,
-//     childrenByKey: _omit(tree.childrenByKey, root),
-//     dirsByKey: {
-//       ...tree.dirsByKey,
-//       [root]: {
-//         ...tree.dirsByKey[root],
-//         shouldLoadNext: true,
-//         hasNext: undefined,
-//       },
-//     },
-//   };
-// }
-
-function addFlatItems(tree: DbKeysTreeModel, root: string, res: DbKeysNodeModel[], visitedRoots: string[] = []) {
-  const item = tree.dirStateByKey[root];
-  if (!item?.isExpanded) {
-    return false;
-  }
-  const children = tree.childrenByKey[root] || [];
-  for (const child of children) {
-    res.push(child);
-    if (child.type == 'dir') {
-      if (visitedRoots.includes(child.key)) {
-        console.warn('Redis: preventing infinite loop for root', child.key);
-        return false;
-      }
-      addFlatItems(tree, child.key, res, [...visitedRoots, root]);
-    }
-  }
-}
-
-export function dbKeys_getFlatList(tree: DbKeysTreeModel) {
-  const res: DbKeysNodeModel[] = [];
-  addFlatItems(tree, '', res);
-  return res;
-}

packages/tools/src/index.ts

@@ -24,6 +24,6 @@ export * from './getConnectionLabel';
 export * from './detectSqlFilterBehaviour';
 export * from './filterBehaviours';
 export * from './schemaInfoTools';
-export * from './dbKeysLoader';
+export * from './redisKeysLoader';
 export * from './rowProgressReporter';
 export * from './diagramTools';

packages/tools/src/nameTools.ts

@@ -146,4 +146,5 @@ export const DATA_FOLDER_NAMES = [
   { name: 'datadeploy', label: 'Data deploy jobs' },
   { name: 'dbcompare', label: 'Database compare jobs' },
   { name: 'apps', label: 'Applications' },
+  { name: 'themes', label: 'Themes' },
 ];

packages/tools/src/redisKeysLoader.ts

@@ -0,0 +1,328 @@
+import _omit from 'lodash/omit';
+import _sortBy from 'lodash/sortBy';
+
+export const DB_KEYS_SHOW_INCREMENT = 100;
+
+export interface RedisNodeModelBase {
+  text?: string;
+  sortKey: string;
+  key: string;
+  count?: number;
+  level: number;
+  keyPath: string[];
+  parentKey: string;
+}
+
+export interface RedisLeafNodeModel extends RedisNodeModelBase {
+  type: 'string' | 'hash' | 'set' | 'list' | 'zset' | 'stream' | 'binary' | 'ReJSON-RL';
+}
+
+export interface RedisFolderNodeModel extends RedisNodeModelBase {
+  // root: string;
+  type: 'dir';
+  // visibleCount?: number;
+  // isExpanded?: boolean;
+}
+
+export interface RedisFolderStateMode {
+  key: string;
+  visibleCount?: number;
+  isExpanded?: boolean;
+}
+
+export interface RedisTreeModel {
+  treeKeySeparator: string;
+  root: RedisFolderNodeModel;
+  dirsByKey: { [key: string]: RedisFolderNodeModel };
+  dirStateByKey: { [key: string]: RedisFolderStateMode };
+  childrenByKey: { [key: string]: RedisNodeModel[] };
+  keyObjectsByKey: { [key: string]: RedisNodeModel };
+  scannedKeys: number;
+  loadCount: number;
+  dbsize: number;
+  cursor: string;
+  loadedAll: boolean;
+  // refreshAll?: boolean;
+}
+
+export type RedisNodeModel = RedisLeafNodeModel | RedisFolderNodeModel;
+
+export interface RedisLoadedModel {
+  key: string;
+
+  type: 'string' | 'hash' | 'set' | 'list' | 'zset' | 'stream' | 'binary' | 'ReJSON-RL';
+  count?: number;
+}
+
+export interface RedisLoadResult {
+  nextCursor: string;
+  keys: RedisLoadedModel[];
+  dbsize: number;
+}
+
+export type RedisChangeModelFunction = (func: (model: RedisTreeModel) => RedisTreeModel, loadNextPage: boolean) => void;
+
+export function redis_mergeNextPage(tree: RedisTreeModel, nextPage: RedisLoadResult): RedisTreeModel {
+  const keyObjectsByKey = { ...tree.keyObjectsByKey };
+
+  for (const keyObj of nextPage.keys) {
+    const keyPath = keyObj.key.split(tree.treeKeySeparator);
+    keyObjectsByKey[keyObj.key] = {
+      ...keyObj,
+      level: keyPath.length,
+      text: keyPath[keyPath.length - 1],
+      sortKey: keyPath[keyPath.length - 1],
+      keyPath,
+      parentKey: keyPath.slice(0, -1).join(tree.treeKeySeparator),
+    };
+  }
+
+  const dirsByKey: { [key: string]: RedisFolderNodeModel } = {};
+  const childrenByKey: { [key: string]: RedisNodeModel[] } = {};
+
+  dirsByKey[''] = tree.root;
+
+  for (const keyObj of Object.values(keyObjectsByKey)) {
+    const dirPath = keyObj.keyPath.slice(0, -1);
+    const dirKey = dirPath.join(tree.treeKeySeparator);
+
+    let dirDepth = keyObj.keyPath.length - 1;
+
+    while (dirDepth > 0) {
+      const newDirPath = keyObj.keyPath.slice(0, dirDepth);
+      const newDirKey = newDirPath.join(tree.treeKeySeparator);
+      if (!dirsByKey[newDirKey]) {
+        dirsByKey[newDirKey] = {
+          level: keyObj.level - 1,
+          keyPath: newDirPath,
+          parentKey: newDirPath.slice(0, -1).join(tree.treeKeySeparator),
+          type: 'dir',
+          key: newDirKey,
+          text: `${newDirPath[newDirPath.length - 1]}${tree.treeKeySeparator}*`,
+          sortKey: newDirPath[newDirPath.length - 1],
+        };
+      }
+
+      dirDepth -= 1;
+    }
+
+    if (!childrenByKey[dirKey]) {
+      childrenByKey[dirKey] = [];
+    }
+
+    childrenByKey[dirKey].push(keyObj);
+  }
+
+  for (const dirObj of Object.values(dirsByKey)) {
+    if (dirObj.key == '') {
+      continue;
+    }
+
+    if (!childrenByKey[dirObj.parentKey]) {
+      childrenByKey[dirObj.parentKey] = [];
+    }
+    childrenByKey[dirObj.parentKey].push(dirObj);
+
+    // set key count
+    dirsByKey[dirObj.key].count = childrenByKey[dirObj.key].length;
+  }
+
+  for (const key in childrenByKey) {
+    childrenByKey[key] = _sortBy(childrenByKey[key], 'sortKey');
+  }
+
+  return {
+    ...tree,
+    cursor: nextPage.nextCursor,
+    dirsByKey,
+    childrenByKey,
+    keyObjectsByKey,
+    scannedKeys: tree.scannedKeys + tree.loadCount,
+    loadedAll: nextPage.nextCursor == '0',
+    dbsize: nextPage.dbsize,
+  };
+}
+
+export function redis_markNodeExpanded(tree: RedisTreeModel, root: string, isExpanded: boolean): RedisTreeModel {
+  const node = tree.dirStateByKey[root];
+  return {
+    ...tree,
+    dirStateByKey: {
+      ...tree.dirStateByKey,
+      [root]: {
+        ...node,
+        isExpanded,
+      },
+    },
+  };
+}
+
+export function redis_showNextItems(tree: RedisTreeModel, root: string): RedisTreeModel {
+  const node = tree.dirStateByKey[root];
+  return {
+    ...tree,
+    dirStateByKey: {
+      ...tree.dirStateByKey,
+      [root]: {
+        ...node,
+        visibleCount: (node?.visibleCount ?? DB_KEYS_SHOW_INCREMENT) + DB_KEYS_SHOW_INCREMENT,
+      },
+    },
+  };
+}
+
+export function redis_createNewModel(treeKeySeparator: string): RedisTreeModel {
+  const root: RedisFolderNodeModel = {
+    level: 0,
+    type: 'dir',
+    keyPath: [],
+    parentKey: '',
+    key: '',
+    sortKey: '',
+  };
+  return {
+    treeKeySeparator,
+    childrenByKey: {},
+    keyObjectsByKey: {},
+    dirsByKey: {
+      '': root,
+    },
+    dirStateByKey: {
+      '': {
+        key: '',
+        visibleCount: DB_KEYS_SHOW_INCREMENT,
+        isExpanded: true,
+      },
+    },
+    scannedKeys: 0,
+    dbsize: 0,
+    loadCount: 2000,
+    cursor: '0',
+    root,
+    loadedAll: false,
+  };
+}
+
+export function redis_clearLoadedData(tree: RedisTreeModel): RedisTreeModel {
+  return {
+    ...tree,
+    childrenByKey: {},
+    keyObjectsByKey: {},
+    dirsByKey: {
+      '': tree.root,
+    },
+    scannedKeys: 0,
+    dbsize: 0,
+    cursor: '0',
+    loadedAll: false,
+  };
+}
+
+function addFlatItems(tree: RedisTreeModel, root: string, res: RedisNodeModel[], visitedRoots: string[] = []) {
+  const item = tree.dirStateByKey[root];
+  if (!item?.isExpanded) {
+    return false;
+  }
+  const children = tree.childrenByKey[root] || [];
+  for (const child of children) {
+    res.push(child);
+    if (child.type == 'dir') {
+      if (visitedRoots.includes(child.key)) {
+        console.warn('Redis: preventing infinite loop for root', child.key);
+        return false;
+      }
+      addFlatItems(tree, child.key, res, [...visitedRoots, root]);
+    }
+  }
+}
+
+export function redis_getFlatList(tree: RedisTreeModel) {
+  const res: RedisNodeModel[] = [];
+  addFlatItems(tree, '', res);
+  return res;
+}
+
+export interface SupportedRedisKeyType {
+  name: string;
+  label: string;
+  dbKeyFields: {
+    name: string;
+    cols?: number;
+    label?: string;
+    placeholder?: string;
+  }[];
+  dbKeyFieldsForGrid?: {
+    name: string;
+    cols?: number;
+    label?: string;
+  }[];
+  keyColumn?: string;
+  showItemList?: boolean;
+  showGeneratedId?: boolean;
+}
+
+export const supportedRedisKeyTypes: SupportedRedisKeyType[] = [
+  {
+    name: 'string',
+    label: 'String',
+    dbKeyFields: [{ name: 'value' }],
+  },
+  {
+    name: 'list',
+    label: 'List',
+    dbKeyFields: [{ name: 'value', cols: 12 }],
+    showItemList: true,
+  },
+  {
+    name: 'set',
+    label: 'Set',
+    dbKeyFields: [{ name: 'value', cols: 12 }],
+    keyColumn: 'value',
+    showItemList: true,
+  },
+  {
+    name: 'zset',
+    label: 'Sorted Set',
+    dbKeyFields: [
+      { name: 'member', cols: 8 },
+      { name: 'score', cols: 4 },
+    ],
+    keyColumn: 'member',
+    showItemList: true,
+  },
+  {
+    name: 'hash',
+    label: 'Hash',
+    dbKeyFields: [
+      { name: 'key', cols: 3, label: 'Field' },
+      { name: 'value', cols: 7 },
+      { name: 'ttl', cols: 2, label: 'TTL' },
+    ],
+    keyColumn: 'key',
+    showItemList: true,
+  },
+  {
+    name: 'stream',
+    label: 'Stream',
+    dbKeyFields: [
+      { name: 'field', cols: 6 },
+      { name: 'value', cols: 6 },
+    ],
+    dbKeyFieldsForGrid: [
+      { name: 'id', cols: 6 },
+      { name: 'value', cols: 6 },
+    ],
+    keyColumn: 'id',
+    showItemList: true,
+    showGeneratedId: true,
+  },
+  {
+    name: 'json',
+    label: 'JSON',
+    dbKeyFields: [{ name: 'value' }],
+  },
+];
+
+export function findSupportedRedisKeyType(type: string): SupportedRedisKeyType | undefined {
+  return supportedRedisKeyTypes.find(t => t.name === type);
+}

packages/tools/src/stringTools.ts

@@ -45,14 +45,15 @@ export function hexStringToArray(inputString) {
 
 export function base64ToHex(base64String) {
   const binaryString = atob(base64String);
-  const hexString = Array.from(binaryString, c =>
-    c.charCodeAt(0).toString(16).padStart(2, '0')
-  ).join('');
+  const hexString = Array.from(binaryString, c => c.charCodeAt(0).toString(16).padStart(2, '0')).join('');
   return '0x' + hexString.toUpperCase();
-};
+}
 
 export function hexToBase64(hexString) {
-  const binaryString = hexString.match(/.{1,2}/g).map(byte => String.fromCharCode(parseInt(byte, 16))).join('');
+  const binaryString = hexString
+    .match(/.{1,2}/g)
+    .map(byte => String.fromCharCode(parseInt(byte, 16)))
+    .join('');
   return btoa(binaryString);
 }
 
@@ -68,9 +69,9 @@ export function parseCellValue(value, editorTypes?: DataEditorTypesBehaviour) {
     if (mHex) {
       return {
         $binary: {
-          base64: hexToBase64(value.substring(2))
-        }
-      }
+          base64: hexToBase64(value.substring(2)),
+        },
+      };
     }
   }
 
@@ -200,6 +201,26 @@ function stringifyJsonToGrid(value): ReturnType<typeof stringifyCellValue> {
   return { value: '(JSON)', gridStyle: 'nullCellStyle' };
 }
 
+function formatNumberCustomSeparator(value, thousandsSeparator) {
+  const [intPart, decPart] = value.split('.');
+  const intPartWithSeparator = intPart.replace(/\B(?=(\d{3})+(?!\d))/g, thousandsSeparator);
+  return decPart ? `${intPartWithSeparator}.${decPart}` : intPartWithSeparator;
+}
+
+function formatCellNumber(value, gridFormattingOptions?: { thousandsSeparator?: string }) {
+  const separator = gridFormattingOptions?.thousandsSeparator;
+  if (_isNumber(value)) {
+    if (separator === 'none' || (value < 1000 && value > -1000)) return value.toString();
+    if (separator === 'system') return value.toLocaleString();
+  }
+  // fallback for system locale
+  if (separator === 'space' || separator === 'system') return formatNumberCustomSeparator(value.toString(), ' ');
+  if (separator === 'narrowspace') return formatNumberCustomSeparator(value.toString(), '\u202F');
+  if (separator === 'comma') return formatNumberCustomSeparator(value.toString(), ',');
+  if (separator === 'dot') return formatNumberCustomSeparator(value.toString(), '.');
+  return value.toString();
+}
+
 export function stringifyCellValue(
   value,
   intent:
@@ -210,7 +231,7 @@ export function stringifyCellValue(
     | 'exportIntent'
     | 'clipboardIntent',
   editorTypes?: DataEditorTypesBehaviour,
-  gridFormattingOptions?: { useThousandsSeparator?: boolean },
+  gridFormattingOptions?: { thousandsSeparator?: string },
   jsonParsedValue?: any
 ): {
   value: string;
@@ -251,12 +272,19 @@ export function stringifyCellValue(
     };
   }
 
+  if (value?.$decimal) {
+    return {
+      value: formatCellNumber(value.$decimal, gridFormattingOptions),
+      gridStyle: 'valueCellStyle',
+    };
+  }
+
   if (editorTypes?.parseHexAsBuffer) {
     // if (value?.type == 'Buffer' && _isArray(value.data)) {
     //   return { value: '0x' + arrayToHexString(value.data), gridStyle: 'valueCellStyle' };
     // }
   }
-  
+
   if (editorTypes?.parseObjectIdAsDollar) {
     if (value?.$oid) {
       switch (intent) {
@@ -270,13 +298,13 @@ export function stringifyCellValue(
   }
   if (value?.$bigint) {
     return {
-      value: value.$bigint,
+      value: formatCellNumber(value.$bigint, gridFormattingOptions),
       gridStyle: 'valueCellStyle',
     };
   }
   if (typeof value === 'bigint') {
     return {
-      value: value.toString(),
+      value: formatCellNumber(value.toString(), gridFormattingOptions),
       gridStyle: 'valueCellStyle',
     };
   }
@@ -351,13 +379,8 @@ export function stringifyCellValue(
   if (_isNumber(value)) {
     switch (intent) {
       case 'gridCellIntent':
-        return {
-          value:
-            gridFormattingOptions?.useThousandsSeparator && (value >= 10000 || value <= -10000)
-              ? value.toLocaleString()
-              : value.toString(),
-          gridStyle: 'valueCellStyle',
-        };
+        const separator = gridFormattingOptions?.thousandsSeparator;
+        return { value: formatCellNumber(value, gridFormattingOptions), gridStyle: 'valueCellStyle' };
       default:
         return { value: value.toString() };
     }
@@ -449,6 +472,9 @@ export function shouldOpenMultilineDialog(value) {
   if (value?.$bigint) {
     return false;
   }
+  if (value?.$decimal) {
+    return false;
+  }
   if (_isPlainObject(value) || _isArray(value)) {
     return true;
   }
@@ -478,6 +504,7 @@ export function getIconForRedisType(type) {
     case 'binary':
       return 'img type-binary';
     case 'ReJSON-RL':
+    case 'JSON':
       return 'img type-rejson';
     default:
       return null;
@@ -699,6 +726,9 @@ export function deserializeJsTypesFromJsonParse(obj) {
     if (value?.$bigint) {
       return BigInt(value.$bigint);
     }
+    if (value?.$decimal) {
+      return value.$decimal;
+    }
   });
 }
 
@@ -713,6 +743,9 @@ export function deserializeJsTypesReviver(key, value) {
   if (value?.$bigint) {
     return BigInt(value.$bigint);
   }
+  if (value?.$decimal) {
+    return value.$decimal;
+  }
   return value;
 }
 

packages/types/dialect.d.ts

@@ -10,6 +10,8 @@ export interface SqlDialect {
   offsetFetchRangeSyntax?: boolean;
   offsetFirstSkipSyntax?: boolean;
   offsetNotSupported?: boolean;
+  useDatalengthForEmptyString?(dataType: string): boolean;
+  disableGroupingForDataType?(dataType: string): boolean;
   quoteIdentifier(s: string): string;
   fallbackDataType?: string;
   explicitDropConstraint?: boolean;
@@ -48,6 +50,12 @@ export interface SqlDialect {
   multipleSchema?: boolean;
   filteredIndexes?: boolean;
   namedDefaultConstraint?: boolean;
+  indexTypes?: {
+    value: string;
+    label: string;
+    isUnique?: boolean;
+    indexType?: string;
+  }[];
 
   specificNullabilityImplementation?: boolean;
   implicitNullDeclaration?: boolean;

packages/types/engines.d.ts

@@ -15,6 +15,8 @@ import {
 } from './dbinfo';
 import { FilterBehaviour } from './filter-type';
 
+export type EngineDriverIcon = string | { light: string; dark?: string };
+
 export interface StreamOptions {
   recordset: (columns) => void;
   row: (row) => void;
@@ -224,6 +226,15 @@ export interface RestoreDatabaseSettings extends BackupRestoreSettingsBase {
   inputFile: string;
 }
 
+export interface DatabaseMethodCallItem {
+  method: string;
+  args: any[];
+}
+
+export interface DatabaseMethodCallList {
+  calls: DatabaseMethodCallItem[];
+}
+
 export interface EngineDriver<TClient = any, TDataBase = any> extends FilterBehaviourProvider {
   engine: string;
   title: string;
@@ -231,7 +242,6 @@ export interface EngineDriver<TClient = any, TDataBase = any> extends FilterBeha
   databaseEngineTypes: string[];
   editorMode?: string;
   readOnlySessions: boolean;
-  supportedKeyTypes: SupportedDbKeyType[];
   dataEditorTypesBehaviour: DataEditorTypesBehaviour;
   supportsDatabaseUrl?: boolean;
   supportsDatabaseBackup?: boolean;
@@ -253,6 +263,7 @@ export interface EngineDriver<TClient = any, TDataBase = any> extends FilterBeha
   collectionPluralLabel?: string;
   collectionNameLabel?: string;
   newCollectionFormParams?: any[];
+  icon?: EngineDriverIcon;
 
   supportedCreateDatabase?: boolean;
   showConnectionField?: (
@@ -337,6 +348,8 @@ export interface EngineDriver<TClient = any, TDataBase = any> extends FilterBeha
   readCollection(dbhan: DatabaseHandle<TClient, TDataBase>, options: ReadCollectionOptions): Promise<any>;
   updateCollection(dbhan: DatabaseHandle<TClient, TDataBase>, changeSet: any): Promise<any>;
   getCollectionUpdateScript(changeSet: any, collectionInfo: CollectionInfo): string;
+  getKeyValueMethodCallList(changeSet: any): DatabaseMethodCallList;
+  invokeMethodCallList(dbhan: DatabaseHandle<TClient, TDataBase>, callList: DatabaseMethodCallList): Promise<void>;
   createDatabase(dbhan: DatabaseHandle<TClient, TDataBase>, name: string): Promise;
   dropDatabase(dbhan: DatabaseHandle<TClient, TDataBase>, name: string): Promise;
   getQuerySplitterOptions(usage: 'stream' | 'script' | 'editor' | 'import'): any;

packages/types/extensions.d.ts

@@ -23,10 +23,12 @@ export interface FileFormatDefinition {
 }
 
 export interface ThemeDefinition {
-  themeClassName: string;
   themeName: string;
   themeType: 'light' | 'dark';
-  themeCss?: string;
+  isBuiltInTheme?: boolean;
+  themeVariables?: { [key: string]: string };
+  themePublicCloudPath?: string;
+  editorTheme?: string;
 }
 
 export interface PluginDefinition {
@@ -47,5 +49,4 @@ export interface ExtensionsDirectory {
   fileFormats: FileFormatDefinition[];
   quickExports: QuickExportDefinition[];
   drivers: EngineDriver[];
-  themes: ThemeDefinition[];
 }

packages/types/package.json

@@ -1,5 +1,5 @@
 {
-  "version": "6.0.0-alpha.1",
+  "version": "7.0.0-alpha.1",
   "name": "dbgate-types",
   "homepage": "https://dbgate.org/",
   "repository": {

packages/web/build-index.js

@@ -13,6 +13,8 @@ for (const page of [
   'admin-license',
   'set-admin-password',
   'redirect',
+  'forgot-password',
+  'reset-password',
 ]) {
   const text = template.replace(/{{page}}/g, page);
   fs.writeFileSync(`public/${page || 'index'}.html`, text);

packages/web/index.html.tpl

@@ -20,6 +20,7 @@
     <link rel="stylesheet" href="dimensions.css" />
     <link rel="stylesheet" href="bulma.css" />
     <link rel="stylesheet" href="icon-colors.css" />
+    <link rel="stylesheet" href="build/tailwind.css" />
     <link rel="stylesheet" href="build/bundle.css" />
     <link rel="stylesheet" href="build/fonts/materialdesignicons.css" />
     <link rel="stylesheet" href="build/diff2html.min.css" />

packages/web/package.json

@@ -1,6 +1,6 @@
 {
   "name": "dbgate-web",
-  "version": "6.0.0-alpha.1",
+  "version": "7.0.0-alpha.1",
   "scripts": {
     "build": "yarn build:index && rollup -c",
     "dev": "yarn build:index && cross-env API_URL=http://localhost:3000 rollup -c -w",
@@ -17,7 +17,6 @@
     "public"
   ],
   "devDependencies": {
-    "@ant-design/colors": "^5.0.0",
     "@energiency/chartjs-plugin-piechart-outlabels": "^1.3.4",
     "@mdi/font": "^7.1.96",
     "@rollup/plugin-commonjs": "^20.0.0",
@@ -25,17 +24,19 @@
     "@rollup/plugin-node-resolve": "^13.0.5",
     "@rollup/plugin-replace": "^3.0.0",
     "@rollup/plugin-typescript": "^8.2.5",
+    "@tailwindcss/postcss": "^4.1.18",
     "@tsconfig/svelte": "^1.0.0",
     "ace-builds": "^1.36.5",
+    "autoprefixer": "^10.4.23",
     "chart.js": "^4.4.2",
     "chartjs-adapter-moment": "^1.0.0",
     "chartjs-plugin-datalabels": "^2.2.0",
     "cross-env": "^7.0.3",
-    "dbgate-datalib": "^6.0.0-alpha.1",
+    "dbgate-datalib": "^7.0.0-alpha.1",
     "dbgate-query-splitter": "^4.11.9",
-    "dbgate-sqltree": "^6.0.0-alpha.1",
-    "dbgate-tools": "^6.0.0-alpha.1",
-    "dbgate-types": "^6.0.0-alpha.1",
+    "dbgate-sqltree": "^7.0.0-alpha.1",
+    "dbgate-tools": "^7.0.0-alpha.1",
+    "dbgate-types": "^7.0.0-alpha.1",
     "diff": "^5.0.0",
     "diff2html": "^3.4.13",
     "file-selector": "^0.2.4",
@@ -44,21 +45,24 @@
     "json-stable-stringify": "^1.0.1",
     "localforage": "^1.9.0",
     "lodash": "^4.17.21",
+    "postcss": "^8.5.6",
     "randomcolor": "^0.6.2",
     "resize-observer-polyfill": "^1.5.1",
     "rollup": "^2.57.0",
     "rollup-plugin-copy": "^3.3.0",
     "rollup-plugin-css-only": "^3.1.0",
     "rollup-plugin-livereload": "^2.0.0",
-    "rollup-plugin-svelte": "^7.0.0",
-    "rollup-plugin-terser": "^7.0.0",
+    "rollup-plugin-postcss": "^4.0.2",
+    "rollup-plugin-svelte": "^7.2.2",
+    "rollup-plugin-terser": "^7.0.2",
     "sirv-cli": "^1.0.0",
     "sql-formatter": "^3.1.0",
-    "svelte": "^3.46.4",
+    "svelte": "^4.2.20",
     "svelte-check": "^1.0.0",
     "svelte-markdown": "^0.1.4",
     "svelte-preprocess": "^4.9.5",
     "svelte-select": "^4.4.7",
+    "tailwindcss": "^4.1.18",
     "tslib": "^2.3.1",
     "typescript": "^4.4.3",
     "uuid": "^3.4.0"

packages/web/postcss.config.js

@@ -0,0 +1,6 @@
+module.exports = {
+  plugins: {
+    '@tailwindcss/postcss': {},
+    autoprefixer: {},
+  },
+};

packages/web/public/dimensions.css

@@ -2,14 +2,21 @@
   --dim-widget-icon-size: 50px;
   --dim-statusbar-height: 22px;
   --dim-left-panel-width: 300px;
+  --dim-right-panel-width: 300px;
   --dim-tabs-height: 33px;
   --dim-tabs-panel-height: calc( var(--dim-visible-tabs-databases) * 20px + var(--dim-tabs-height) );
-  --dim-splitter-thickness: 3px;
+  --dim-splitter-thickness: 4px;
 
   --dim-visible-left-panel: 1; /* set from JS */
   --dim-content-left: calc(
     var(--dim-widget-icon-size) + var(--dim-visible-left-panel) *
-      (var(--dim-left-panel-width) + var(--dim-splitter-thickness))
+      (var(--dim-left-panel-width))
+  );
+
+  --dim-visible-right-panel: 0; /* set from JS */
+  --dim-content-right: calc(
+    var(--dim-visible-right-panel) *
+      (var(--dim-right-panel-width))
   );
 
   --dim-visible-toolbar: 0; /* set from JS */

packages/web/public/global.css

@@ -12,21 +12,47 @@ body {
 }
 
 .horizontal-split-handle {
-  background-color: var(--theme-border);
-  width: var(--dim-splitter-thickness);
+  width: 0px;
+  position: relative;
   cursor: col-resize;
 }
-.horizontal-split-handle:hover {
-  background-color: var(--theme-bg-2);
+.horizontal-split-handle::before {
+  content: '';
+  position: absolute;
+  cursor: col-resize;
+  left: -1px;
+  right: -1px;
+  top: 0;
+  bottom: 0;
+  width: var(--dim-splitter-thickness);
+  background-color: transparent;
+  transition: background-color 0.2s ease;
+  z-index: 200;
+}
+.horizontal-split-handle:hover::before {
+  background-color: var(--theme-splitter-active);
 }
 
 .vertical-split-handle {
-  background-color: var(--theme-border);
-  height: var(--dim-splitter-thickness);
+  height: 0px;
+  position: relative;
   cursor: row-resize;
 }
-.vertical-split-handle:hover {
-  background-color: var(--theme-bg-2);
+.vertical-split-handle::before {
+  content: '';
+  position: absolute;
+  cursor: row-resize;
+  top: -1px;
+  bottom: -1px;
+  left: 0;
+  right: 0;
+  height: var(--dim-splitter-thickness);
+  background-color: transparent;
+  transition: background-color 0.2s ease;
+  z-index: 200;
+}
+.vertical-split-handle:hover::before {
+  background-color: var(--theme-splitter-active);
 }
 
 .icon-invisible {
@@ -65,19 +91,7 @@ body {
   overflow: scroll;
 }
 .bg-0 {
-  background-color: var(--theme-bg-0);
-}
-.bg-1 {
-  background-color: var(--theme-bg-1);
-}
-.bg-2 {
-  background-color: var(--theme-bg-2);
-}
-.bg-3 {
-  background-color: var(--theme-bg-3);
-}
-.bg-4 {
-  background-color: var(--theme-bg-4);
+  background-color: var(--theme-content-background);
 }
 
 .col-10 {
@@ -117,21 +131,32 @@ body {
   max-width: 16.6666%;
 }
 
-.largeFormMarker input[type='text'], .largeFormMarker input[type='number'], .largeFormMarker input[type='password'], .largeFormMarker textarea  {
+.largeFormMarker input[type='text'],
+.largeFormMarker input[type='number'],
+.largeFormMarker input[type='password'],
+.largeFormMarker textarea {
   width: 100%;
   padding: 10px 10px;
   font-size: 14px;
   box-sizing: border-box;
   border-radius: 4px;
-  border: 1px solid var(--theme-border);
+  border: var(--theme-input-border);
+  background: var(--theme-input-background);
 }
 
 .input1 {
-  padding: 5px 5px;
+  padding: 5px 2px;
   font-size: 14px;
   box-sizing: border-box;
   border-radius: 4px;
-  border: 1px solid var(--theme-border);
+  border: var(--theme-input-border);
+  background: var(--theme-input-background);
+}
+
+.input1:focus {
+  outline: none;
+  border: var(--theme-input-border-focus);
+  box-shadow: var(--theme-input-focus-ring);
 }
 
 .largeFormMarker select {
@@ -148,59 +173,94 @@ body *::-webkit-scrollbar {
 }
 body *::-webkit-scrollbar-track {
   border-radius: 1px;
-  background-color: var(--theme-bg-1);
+  background: var(--theme-scrollbar-background);
 }
 body *::-webkit-scrollbar-corner {
   border-radius: 1px;
-  background-color: var(--theme-bg-2);
+  background: var(--theme-scrollbar-corner-background);
 }
 
 body *::-webkit-scrollbar-thumb {
   border-radius: 1px;
-  background-color: var(--theme-bg-3);
+  background: var(--theme-scrollbar-thumb-background);
 }
 
 body *::-webkit-scrollbar-thumb:hover {
-  background-color: var(--theme-bg-4);
+  background: var(--theme-scrollbar-thumb-background-hover);
 }
 
 input {
-  background-color: var(--theme-bg-0);
-  color: var(--theme-font-1);
-  border: 1px solid var(--theme-border);
+  background: var(--theme-input-background);
+  color: var(--theme-generic-font);
+  border: var(--theme-input-border);
 }
 
 input[disabled] {
-  background-color: var(--theme-bg-1);
+  background: var(--theme-input-background-disabled);
+  color: var(--theme-input-foreground-disabled);  
+}
+
+.largeFormMarker input[disabled] {
+  background: var(--theme-input-background-disabled);
+  color: var(--theme-input-foreground-disabled);  
 }
 
 select {
-  background-color: var(--theme-bg-0);
-  color: var(--theme-font-1);
-  border: 1px solid var(--theme-border);
+  padding: 10px 12px;
+    border: var(--theme-input-border);
+    border-radius: 4px;
+    background-color: var(--theme-input-background);
+    color: var(--theme-input-foreground);
+    font-size: 13px;
+    transition: all 0.15s ease;
+    font-family: inherit;
+}
+
+select:hover {
+  border: var(--theme-input-border-hover);
+}
+
+select:focus {
+  outline: none;
+  border: var(--theme-input-border-focus);
+  box-shadow: var(--theme-input-focus-ring);
 }
 
 select[disabled] {
-  background-color: var(--theme-bg-1);
+  background-color: var(--theme-input-background-disabled);
+  color: var(--theme-input-foreground-disabled);
+  cursor: not-allowed;
+  border: var(--theme-input-border-disabled);
+}
+
+.largeFormMarker select[disabled] {
+  background: var(--theme-input-background-disabled);
+  color: var(--theme-input-foreground-disabled);  
 }
 
 .classicform select {
   padding: 5px 5px 4px;
 }
 
+.selectContainer.focused {
+  box-shadow: var(--theme-input-focus-ring);
+}
+
 textarea {
-  background-color: var(--theme-bg-0);
-  color: var(--theme-font-1);
-  border: 1px solid var(--theme-border);
+  background: var(--theme-input-background);
+  color: var(--theme-generic-font);
+  border: var(--theme-input-border);
 }
 
 textarea[disabled] {
-  background-color: var(--theme-bg-1);
+  background: var(--theme-input-background-disabled);
+  color: var(--theme-input-foreground-disabled);
 }
 
 .ace_gutter-cell.ace-gutter-sql-run {
   background-repeat: no-repeat;
   background-position: 2px center;
+  background-size: 12px 12px;
 
   /* content: '▶';
   margin-right: 3px; */
@@ -219,11 +279,69 @@ textarea[disabled] {
 }
 
 .ace_gutter-cell.ace-gutter-sql-run:hover {
-   background-color: var(--theme-bg-2);
+  background-color: var(--theme-datagrid-cell-background-alt);
 }
 
 .ace_gutter-cell.ace-gutter-current-part {
-   /* background-color: var(--theme-bg-2); */
-   font-weight: bold;
-   color: var(--theme-font-hover);
-}
+  font-weight: bold;
+  color: var(--theme-generic-font-hover);
+}
+
+input[type='checkbox'] {
+  appearance: none;
+  width: 16px;
+  height: 16px;
+
+  border-radius: 4px;
+  border: var(--theme-checkbox-border);
+  background-color: var(--theme-input-background);
+
+  display: inline-grid;
+  place-content: center;
+
+  cursor: pointer;
+}
+
+input[type='checkbox']:hover:not(:disabled) {
+  border: 1px solid var(--theme-checkbox-hover-not-disabled);
+}
+
+input[type='checkbox']:checked:hover {
+  border: 1px solid var(--theme-checkbox-background);
+}
+
+input[type='checkbox']:checked {
+  background-color: var(--theme-checkbox-background);
+  border: var(--theme-checkbox-border);
+}
+
+input[type='checkbox']::before {
+  content: '';
+  width: 10px;
+  height: 10px;
+  transform: scale(0);
+  transition: transform 120ms ease;
+  background-color: var(--theme-checkbox-mark);
+  clip-path: polygon(
+    14% 44%,
+    0 65%,
+    50% 100%,
+    100% 16%,
+    80% 0,
+    43% 62%
+  );
+}
+
+input[type='checkbox']:checked::before {
+  transform: scale(1);
+}
+
+input[type='checkbox']:disabled {
+  cursor: not-allowed;
+  background-color: var(--theme-checkbox-background-disabled);
+  border: var(--theme-checkbox-border);
+}
+
+input[type='checkbox']:disabled::before {
+  background-color: var(--theme-checkbox-background-disabled-before);
+}

packages/web/public/icon-colors.css

@@ -23,12 +23,12 @@
 }
 
 
-.color-icon-inv-green {
-  color: var(--theme-icon-inv-green);
+.color-icon-statusbar-red {
+  color: var(--theme-statusbar-icon-error);
 }
 
-.color-icon-inv-red {
-  color: var(--theme-icon-inv-red);
+.color-icon-statusbar-green {
+  color: var(--theme-statusbar-icon-ok);
 }
 
 .premium-background-gradient {

packages/web/public/tailwind-colors.css

@@ -0,0 +1,288 @@
+/* Force all Tailwind color variables to be included in build */
+body {
+    /* Slate */
+    --tw-copy-slate-50: var(--tw-color-slate-50);
+    --tw-copy-slate-100: var(--tw-color-slate-100);
+    --tw-copy-slate-200: var(--tw-color-slate-200);
+    --tw-copy-slate-300: var(--tw-color-slate-300);
+    --tw-copy-slate-400: var(--tw-color-slate-400);
+    --tw-copy-slate-500: var(--tw-color-slate-500);
+    --tw-copy-slate-600: var(--tw-color-slate-600);
+    --tw-copy-slate-700: var(--tw-color-slate-700);
+    --tw-copy-slate-800: var(--tw-color-slate-800);
+    --tw-copy-slate-900: var(--tw-color-slate-900);
+    --tw-copy-slate-950: var(--tw-color-slate-950);
+    
+    /* Gray */
+    --tw-copy-gray-50: var(--tw-color-gray-50);
+    --tw-copy-gray-100: var(--tw-color-gray-100);
+    --tw-copy-gray-200: var(--tw-color-gray-200);
+    --tw-copy-gray-300: var(--tw-color-gray-300);
+    --tw-copy-gray-400: var(--tw-color-gray-400);
+    --tw-copy-gray-500: var(--tw-color-gray-500);
+    --tw-copy-gray-600: var(--tw-color-gray-600);
+    --tw-copy-gray-700: var(--tw-color-gray-700);
+    --tw-copy-gray-800: var(--tw-color-gray-800);
+    --tw-copy-gray-900: var(--tw-color-gray-900);
+    --tw-copy-gray-950: var(--tw-color-gray-950);
+    
+    /* Zinc */
+    --tw-copy-zinc-50: var(--tw-color-zinc-50);
+    --tw-copy-zinc-100: var(--tw-color-zinc-100);
+    --tw-copy-zinc-200: var(--tw-color-zinc-200);
+    --tw-copy-zinc-300: var(--tw-color-zinc-300);
+    --tw-copy-zinc-400: var(--tw-color-zinc-400);
+    --tw-copy-zinc-500: var(--tw-color-zinc-500);
+    --tw-copy-zinc-600: var(--tw-color-zinc-600);
+    --tw-copy-zinc-700: var(--tw-color-zinc-700);
+    --tw-copy-zinc-800: var(--tw-color-zinc-800);
+    --tw-copy-zinc-900: var(--tw-color-zinc-900);
+    --tw-copy-zinc-950: var(--tw-color-zinc-950);
+    
+    /* Neutral */
+    --tw-copy-neutral-50: var(--tw-color-neutral-50);
+    --tw-copy-neutral-100: var(--tw-color-neutral-100);
+    --tw-copy-neutral-200: var(--tw-color-neutral-200);
+    --tw-copy-neutral-300: var(--tw-color-neutral-300);
+    --tw-copy-neutral-400: var(--tw-color-neutral-400);
+    --tw-copy-neutral-500: var(--tw-color-neutral-500);
+    --tw-copy-neutral-600: var(--tw-color-neutral-600);
+    --tw-copy-neutral-700: var(--tw-color-neutral-700);
+    --tw-copy-neutral-800: var(--tw-color-neutral-800);
+    --tw-copy-neutral-900: var(--tw-color-neutral-900);
+    --tw-copy-neutral-950: var(--tw-color-neutral-950);
+    
+    /* Stone */
+    --tw-copy-stone-50: var(--tw-color-stone-50);
+    --tw-copy-stone-100: var(--tw-color-stone-100);
+    --tw-copy-stone-200: var(--tw-color-stone-200);
+    --tw-copy-stone-300: var(--tw-color-stone-300);
+    --tw-copy-stone-400: var(--tw-color-stone-400);
+    --tw-copy-stone-500: var(--tw-color-stone-500);
+    --tw-copy-stone-600: var(--tw-color-stone-600);
+    --tw-copy-stone-700: var(--tw-color-stone-700);
+    --tw-copy-stone-800: var(--tw-color-stone-800);
+    --tw-copy-stone-900: var(--tw-color-stone-900);
+    --tw-copy-stone-950: var(--tw-color-stone-950);
+    
+    /* Red */
+    --tw-copy-red-50: var(--tw-color-red-50);
+    --tw-copy-red-100: var(--tw-color-red-100);
+    --tw-copy-red-200: var(--tw-color-red-200);
+    --tw-copy-red-300: var(--tw-color-red-300);
+    --tw-copy-red-400: var(--tw-color-red-400);
+    --tw-copy-red-500: var(--tw-color-red-500);
+    --tw-copy-red-600: var(--tw-color-red-600);
+    --tw-copy-red-700: var(--tw-color-red-700);
+    --tw-copy-red-800: var(--tw-color-red-800);
+    --tw-copy-red-900: var(--tw-color-red-900);
+    --tw-copy-red-950: var(--tw-color-red-950);
+    
+    /* Orange */
+    --tw-copy-orange-50: var(--tw-color-orange-50);
+    --tw-copy-orange-100: var(--tw-color-orange-100);
+    --tw-copy-orange-200: var(--tw-color-orange-200);
+    --tw-copy-orange-300: var(--tw-color-orange-300);
+    --tw-copy-orange-400: var(--tw-color-orange-400);
+    --tw-copy-orange-500: var(--tw-color-orange-500);
+    --tw-copy-orange-600: var(--tw-color-orange-600);
+    --tw-copy-orange-700: var(--tw-color-orange-700);
+    --tw-copy-orange-800: var(--tw-color-orange-800);
+    --tw-copy-orange-900: var(--tw-color-orange-900);
+    --tw-copy-orange-950: var(--tw-color-orange-950);
+    
+    /* Amber */
+    --tw-copy-amber-50: var(--tw-color-amber-50);
+    --tw-copy-amber-100: var(--tw-color-amber-100);
+    --tw-copy-amber-200: var(--tw-color-amber-200);
+    --tw-copy-amber-300: var(--tw-color-amber-300);
+    --tw-copy-amber-400: var(--tw-color-amber-400);
+    --tw-copy-amber-500: var(--tw-color-amber-500);
+    --tw-copy-amber-600: var(--tw-color-amber-600);
+    --tw-copy-amber-700: var(--tw-color-amber-700);
+    --tw-copy-amber-800: var(--tw-color-amber-800);
+    --tw-copy-amber-900: var(--tw-color-amber-900);
+    --tw-copy-amber-950: var(--tw-color-amber-950);
+    
+    /* Yellow */
+    --tw-copy-yellow-50: var(--tw-color-yellow-50);
+    --tw-copy-yellow-100: var(--tw-color-yellow-100);
+    --tw-copy-yellow-200: var(--tw-color-yellow-200);
+    --tw-copy-yellow-300: var(--tw-color-yellow-300);
+    --tw-copy-yellow-400: var(--tw-color-yellow-400);
+    --tw-copy-yellow-500: var(--tw-color-yellow-500);
+    --tw-copy-yellow-600: var(--tw-color-yellow-600);
+    --tw-copy-yellow-700: var(--tw-color-yellow-700);
+    --tw-copy-yellow-800: var(--tw-color-yellow-800);
+    --tw-copy-yellow-900: var(--tw-color-yellow-900);
+    --tw-copy-yellow-950: var(--tw-color-yellow-950);
+    
+    /* Lime */
+    --tw-copy-lime-50: var(--tw-color-lime-50);
+    --tw-copy-lime-100: var(--tw-color-lime-100);
+    --tw-copy-lime-200: var(--tw-color-lime-200);
+    --tw-copy-lime-300: var(--tw-color-lime-300);
+    --tw-copy-lime-400: var(--tw-color-lime-400);
+    --tw-copy-lime-500: var(--tw-color-lime-500);
+    --tw-copy-lime-600: var(--tw-color-lime-600);
+    --tw-copy-lime-700: var(--tw-color-lime-700);
+    --tw-copy-lime-800: var(--tw-color-lime-800);
+    --tw-copy-lime-900: var(--tw-color-lime-900);
+    --tw-copy-lime-950: var(--tw-color-lime-950);
+    
+    /* Green */
+    --tw-copy-green-50: var(--tw-color-green-50);
+    --tw-copy-green-100: var(--tw-color-green-100);
+    --tw-copy-green-200: var(--tw-color-green-200);
+    --tw-copy-green-300: var(--tw-color-green-300);
+    --tw-copy-green-400: var(--tw-color-green-400);
+    --tw-copy-green-500: var(--tw-color-green-500);
+    --tw-copy-green-600: var(--tw-color-green-600);
+    --tw-copy-green-700: var(--tw-color-green-700);
+    --tw-copy-green-800: var(--tw-color-green-800);
+    --tw-copy-green-900: var(--tw-color-green-900);
+    --tw-copy-green-950: var(--tw-color-green-950);
+    
+    /* Emerald */
+    --tw-copy-emerald-50: var(--tw-color-emerald-50);
+    --tw-copy-emerald-100: var(--tw-color-emerald-100);
+    --tw-copy-emerald-200: var(--tw-color-emerald-200);
+    --tw-copy-emerald-300: var(--tw-color-emerald-300);
+    --tw-copy-emerald-400: var(--tw-color-emerald-400);
+    --tw-copy-emerald-500: var(--tw-color-emerald-500);
+    --tw-copy-emerald-600: var(--tw-color-emerald-600);
+    --tw-copy-emerald-700: var(--tw-color-emerald-700);
+    --tw-copy-emerald-800: var(--tw-color-emerald-800);
+    --tw-copy-emerald-900: var(--tw-color-emerald-900);
+    --tw-copy-emerald-950: var(--tw-color-emerald-950);
+    
+    /* Teal */
+    --tw-copy-teal-50: var(--tw-color-teal-50);
+    --tw-copy-teal-100: var(--tw-color-teal-100);
+    --tw-copy-teal-200: var(--tw-color-teal-200);
+    --tw-copy-teal-300: var(--tw-color-teal-300);
+    --tw-copy-teal-400: var(--tw-color-teal-400);
+    --tw-copy-teal-500: var(--tw-color-teal-500);
+    --tw-copy-teal-600: var(--tw-color-teal-600);
+    --tw-copy-teal-700: var(--tw-color-teal-700);
+    --tw-copy-teal-800: var(--tw-color-teal-800);
+    --tw-copy-teal-900: var(--tw-color-teal-900);
+    --tw-copy-teal-950: var(--tw-color-teal-950);
+    
+    /* Cyan */
+    --tw-copy-cyan-50: var(--tw-color-cyan-50);
+    --tw-copy-cyan-100: var(--tw-color-cyan-100);
+    --tw-copy-cyan-200: var(--tw-color-cyan-200);
+    --tw-copy-cyan-300: var(--tw-color-cyan-300);
+    --tw-copy-cyan-400: var(--tw-color-cyan-400);
+    --tw-copy-cyan-500: var(--tw-color-cyan-500);
+    --tw-copy-cyan-600: var(--tw-color-cyan-600);
+    --tw-copy-cyan-700: var(--tw-color-cyan-700);
+    --tw-copy-cyan-800: var(--tw-color-cyan-800);
+    --tw-copy-cyan-900: var(--tw-color-cyan-900);
+    --tw-copy-cyan-950: var(--tw-color-cyan-950);
+    
+    /* Sky */
+    --tw-copy-sky-50: var(--tw-color-sky-50);
+    --tw-copy-sky-100: var(--tw-color-sky-100);
+    --tw-copy-sky-200: var(--tw-color-sky-200);
+    --tw-copy-sky-300: var(--tw-color-sky-300);
+    --tw-copy-sky-400: var(--tw-color-sky-400);
+    --tw-copy-sky-500: var(--tw-color-sky-500);
+    --tw-copy-sky-600: var(--tw-color-sky-600);
+    --tw-copy-sky-700: var(--tw-color-sky-700);
+    --tw-copy-sky-800: var(--tw-color-sky-800);
+    --tw-copy-sky-900: var(--tw-color-sky-900);
+    --tw-copy-sky-950: var(--tw-color-sky-950);
+    
+    /* Blue */
+    --tw-copy-blue-50: var(--tw-color-blue-50);
+    --tw-copy-blue-100: var(--tw-color-blue-100);
+    --tw-copy-blue-200: var(--tw-color-blue-200);
+    --tw-copy-blue-300: var(--tw-color-blue-300);
+    --tw-copy-blue-400: var(--tw-color-blue-400);
+    --tw-copy-blue-500: var(--tw-color-blue-500);
+    --tw-copy-blue-600: var(--tw-color-blue-600);
+    --tw-copy-blue-700: var(--tw-color-blue-700);
+    --tw-copy-blue-800: var(--tw-color-blue-800);
+    --tw-copy-blue-900: var(--tw-color-blue-900);
+    --tw-copy-blue-950: var(--tw-color-blue-950);
+    
+    /* Indigo */
+    --tw-copy-indigo-50: var(--tw-color-indigo-50);
+    --tw-copy-indigo-100: var(--tw-color-indigo-100);
+    --tw-copy-indigo-200: var(--tw-color-indigo-200);
+    --tw-copy-indigo-300: var(--tw-color-indigo-300);
+    --tw-copy-indigo-400: var(--tw-color-indigo-400);
+    --tw-copy-indigo-500: var(--tw-color-indigo-500);
+    --tw-copy-indigo-600: var(--tw-color-indigo-600);
+    --tw-copy-indigo-700: var(--tw-color-indigo-700);
+    --tw-copy-indigo-800: var(--tw-color-indigo-800);
+    --tw-copy-indigo-900: var(--tw-color-indigo-900);
+    --tw-copy-indigo-950: var(--tw-color-indigo-950);
+    
+    /* Violet */
+    --tw-copy-violet-50: var(--tw-color-violet-50);
+    --tw-copy-violet-100: var(--tw-color-violet-100);
+    --tw-copy-violet-200: var(--tw-color-violet-200);
+    --tw-copy-violet-300: var(--tw-color-violet-300);
+    --tw-copy-violet-400: var(--tw-color-violet-400);
+    --tw-copy-violet-500: var(--tw-color-violet-500);
+    --tw-copy-violet-600: var(--tw-color-violet-600);
+    --tw-copy-violet-700: var(--tw-color-violet-700);
+    --tw-copy-violet-800: var(--tw-color-violet-800);
+    --tw-copy-violet-900: var(--tw-color-violet-900);
+    --tw-copy-violet-950: var(--tw-color-violet-950);
+    
+    /* Purple */
+    --tw-copy-purple-50: var(--tw-color-purple-50);
+    --tw-copy-purple-100: var(--tw-color-purple-100);
+    --tw-copy-purple-200: var(--tw-color-purple-200);
+    --tw-copy-purple-300: var(--tw-color-purple-300);
+    --tw-copy-purple-400: var(--tw-color-purple-400);
+    --tw-copy-purple-500: var(--tw-color-purple-500);
+    --tw-copy-purple-600: var(--tw-color-purple-600);
+    --tw-copy-purple-700: var(--tw-color-purple-700);
+    --tw-copy-purple-800: var(--tw-color-purple-800);
+    --tw-copy-purple-900: var(--tw-color-purple-900);
+    --tw-copy-purple-950: var(--tw-color-purple-950);
+    
+    /* Fuchsia */
+    --tw-copy-fuchsia-50: var(--tw-color-fuchsia-50);
+    --tw-copy-fuchsia-100: var(--tw-color-fuchsia-100);
+    --tw-copy-fuchsia-200: var(--tw-color-fuchsia-200);
+    --tw-copy-fuchsia-300: var(--tw-color-fuchsia-300);
+    --tw-copy-fuchsia-400: var(--tw-color-fuchsia-400);
+    --tw-copy-fuchsia-500: var(--tw-color-fuchsia-500);
+    --tw-copy-fuchsia-600: var(--tw-color-fuchsia-600);
+    --tw-copy-fuchsia-700: var(--tw-color-fuchsia-700);
+    --tw-copy-fuchsia-800: var(--tw-color-fuchsia-800);
+    --tw-copy-fuchsia-900: var(--tw-color-fuchsia-900);
+    --tw-copy-fuchsia-950: var(--tw-color-fuchsia-950);
+    
+    /* Pink */
+    --tw-copy-pink-50: var(--tw-color-pink-50);
+    --tw-copy-pink-100: var(--tw-color-pink-100);
+    --tw-copy-pink-200: var(--tw-color-pink-200);
+    --tw-copy-pink-300: var(--tw-color-pink-300);
+    --tw-copy-pink-400: var(--tw-color-pink-400);
+    --tw-copy-pink-500: var(--tw-color-pink-500);
+    --tw-copy-pink-600: var(--tw-color-pink-600);
+    --tw-copy-pink-700: var(--tw-color-pink-700);
+    --tw-copy-pink-800: var(--tw-color-pink-800);
+    --tw-copy-pink-900: var(--tw-color-pink-900);
+    --tw-copy-pink-950: var(--tw-color-pink-950);
+    
+    /* Rose */
+    --tw-copy-rose-50: var(--tw-color-rose-50);
+    --tw-copy-rose-100: var(--tw-color-rose-100);
+    --tw-copy-rose-200: var(--tw-color-rose-200);
+    --tw-copy-rose-300: var(--tw-color-rose-300);
+    --tw-copy-rose-400: var(--tw-color-rose-400);
+    --tw-copy-rose-500: var(--tw-color-rose-500);
+    --tw-copy-rose-600: var(--tw-color-rose-600);
+    --tw-copy-rose-700: var(--tw-color-rose-700);
+    --tw-copy-rose-800: var(--tw-color-rose-800);
+    --tw-copy-rose-900: var(--tw-color-rose-900);
+    --tw-copy-rose-950: var(--tw-color-rose-950);
+}

packages/web/rollup.config.js

@@ -9,6 +9,7 @@ import typescript from '@rollup/plugin-typescript';
 import replace from '@rollup/plugin-replace';
 import css from 'rollup-plugin-css-only';
 import json from '@rollup/plugin-json';
+import postcss from 'rollup-plugin-postcss';
 
 const production = !process.env.ROLLUP_WATCH;
 
@@ -34,6 +35,21 @@ function serve() {
 }
 
 export default [
+  // Separate entry for Tailwind CSS processing
+  {
+    input: 'src/tailwind.css',
+    output: {
+      file: 'public/build/tailwind.css',
+    },
+    plugins: [
+      postcss({
+        extract: true,
+        minimize: production,
+        sourceMap: !production,
+      }),
+    ],
+  },
+
   {
     input: 'src/query/QueryParserWorker.js',
     output: {

packages/web/src/DragAndDropFileTarget.svelte

@@ -41,7 +41,7 @@
     left: 0;
     right: 0;
     bottom: 0;
-    background: var(--theme-bg-selected);
+    background: var(--theme-table-selected-background);
     align-items: center;
     justify-content: space-around;
     z-index: 1000;

packages/web/src/ForgotPasswordPage.svelte

@@ -0,0 +1,119 @@
+<script lang="ts">
+  import { writable } from 'svelte/store';
+  import FormTextField from './forms/FormTextField.svelte';
+  import FormSubmit from './forms/FormSubmit.svelte';
+  import SpecialPageLayout from './widgets/SpecialPageLayout.svelte';
+  import FormProviderCore from './forms/FormProviderCore.svelte';
+  import { apiCall } from './utility/api';
+  import ErrorInfo from './elements/ErrorInfo.svelte';
+  import Link from './elements/Link.svelte';
+
+  const values = writable({});
+
+  let error = null;
+  let success = false;
+  let isSubmitting = false;
+</script>
+
+<SpecialPageLayout>
+  <FormProviderCore {values}>
+    <div class="heading">Reset Password</div>
+
+    {#if success}
+      <div class="success-message">
+        If an account with that email exists, we've sent a password reset link to your email address. Please check your
+        inbox and follow the instructions.
+      </div>
+      <div class="back-link">
+        <Link internalRedirect="/login.html" data-testid="ForgotPasswordPage_backToLogin">
+          Back to Login
+        </Link>
+      </div>
+    {:else}
+      <div class="text">
+        Enter your email address and we'll send you a link to reset your password.
+      </div>
+
+      <FormTextField
+        label="Email"
+        name="email"
+        type="email"
+        autocomplete="email"
+        saveOnInput
+        data-testid="ForgotPasswordPage_email"
+      />
+
+      {#if error}
+        <ErrorInfo message={error} />
+      {/if}
+
+      <div class="submit">
+        <FormSubmit
+          value={isSubmitting ? 'Sending...' : 'Send Reset Link'}
+          disabled={isSubmitting}
+          on:click={async e => {
+            error = null;
+            isSubmitting = true;
+            const resp = await apiCall('storage/request-password-reset', e.detail);
+            isSubmitting = false;
+            if (resp?.error) {
+              error = resp.error;
+              return;
+            }
+            success = true;
+          }}
+          data-testid="ForgotPasswordPage_submit"
+        />
+      </div>
+
+      <div class="back-link">
+        <Link internalRedirect="/login.html" data-testid="ForgotPasswordPage_backToLogin">
+          Back to Login
+        </Link>
+      </div>
+    {/if}
+  </FormProviderCore>
+</SpecialPageLayout>
+
+<style>
+  .heading {
+    text-align: center;
+    margin: 1em;
+    font-size: xx-large;
+  }
+
+  .submit {
+    margin: var(--dim-large-form-margin);
+    display: flex;
+  }
+
+  .submit :global(input) {
+    flex: 1;
+    font-size: larger;
+  }
+
+  .submit :global(input:disabled) {
+    opacity: 0.6;
+    cursor: not-allowed;
+  }
+
+  .text {
+    margin-left: var(--dim-large-form-margin);
+    margin-right: var(--dim-large-form-margin);
+    margin-bottom: var(--dim-large-form-margin);
+  }
+
+  .success-message {
+    margin: var(--dim-large-form-margin);
+    padding: 15px;
+    background-color: var(--theme-bg-green);
+    border: 1px solid var(--theme-border);
+    border-radius: 4px;
+    color: var(--theme-font-1);
+  }
+
+  .back-link {
+    margin: var(--dim-large-form-margin);
+    text-align: center;
+  }
+</style>

packages/web/src/LoginPage.svelte

@@ -5,7 +5,7 @@
   import FormSubmit from './forms/FormSubmit.svelte';
   import FormTextField from './forms/FormTextField.svelte';
   import { apiCall, enableApi, strmid } from './utility/api';
-  import { useConfig } from './utility/metadataLoaders';
+  import { useConfig, useSettings } from './utility/metadataLoaders';
   import ErrorInfo from './elements/ErrorInfo.svelte';
   import FormSelectField from './forms/FormSelectField.svelte';
   import { writable } from 'svelte/store';
@@ -18,6 +18,7 @@
   export let isAdminPage;
 
   const config = useConfig();
+  const settings = useSettings();
 
   let availableConnections = null;
   let availableProviders = [];
@@ -199,6 +200,13 @@
           saveOnInput
           data-testid="LoginPage_password"
         />
+        {#if selectedProvider?.type == 'local' && $settings?.['storage.allowForgottenPasswordReset']}
+          <div class="forgot-password-link">
+            <Link internalRedirect="/forgot-password.html" data-testid="LoginPage_forgotPassword">
+              Don't remember your password?
+            </Link>
+          </div>
+        {/if}
       {/if}
     {/if}
 
@@ -335,13 +343,13 @@
     border-radius: 5px;
     cursor: pointer;
 
-    border: 1px solid var(--theme-bg-button-inv-3);
-    background-color: var(--theme-bg-button-inv-2);
-    color: var(--theme-font-inv-1);
+    border: var(--theme-formbutton-border);
+    background: var(--theme-formbutton-background);
+    color: var(--theme-formbutton-foreground);
   }
 
   .loginButton:hover {
-    background-color: var(--theme-bg-button-inv-3);
+    background: var(--theme-formbutton-background-hover);
   }
 
   .login-link {
@@ -349,4 +357,11 @@
     top: 10px;
     right: 10px;
   }
+
+  .forgot-password-link {
+    margin: var(--dim-large-form-margin);
+    margin-top: 5px;
+    margin-bottom: 0;
+    font-size: 0.9em;
+  }
 </style>

packages/web/src/ResetPasswordPage.svelte

@@ -0,0 +1,157 @@
+<script lang="ts">
+  import { writable } from 'svelte/store';
+  import { onMount } from 'svelte';
+  import FormPasswordField from './forms/FormPasswordField.svelte';
+  import FormSubmit from './forms/FormSubmit.svelte';
+  import SpecialPageLayout from './widgets/SpecialPageLayout.svelte';
+  import FormProviderCore from './forms/FormProviderCore.svelte';
+  import { apiCall } from './utility/api';
+  import ErrorInfo from './elements/ErrorInfo.svelte';
+  import Link from './elements/Link.svelte';
+
+  const values = writable({});
+
+  let error = null;
+  let success = false;
+  let isSubmitting = false;
+  let token = null;
+
+  onMount(() => {
+    const urlParams = new URLSearchParams(window.location.search);
+    token = urlParams.get('token');
+    
+    if (!token) {
+      error = 'Invalid reset link';
+    }
+  });
+</script>
+
+<SpecialPageLayout>
+  <FormProviderCore {values}>
+    <div class="heading">Set New Password</div>
+
+    {#if success}
+      <div class="success-message">
+        Your password has been reset successfully! You can now log in with your new password.
+      </div>
+      <div class="back-link">
+        <Link internalRedirect="/login.html" data-testid="ResetPasswordPage_backToLogin">
+          Go to Login
+        </Link>
+      </div>
+    {:else}
+      <div class="text">
+        Enter your new password below.
+      </div>
+
+      <FormPasswordField
+        label="New Password"
+        name="newPassword"
+        autocomplete="new-password"
+        saveOnInput
+        data-testid="ResetPasswordPage_newPassword"
+      />
+
+      <FormPasswordField
+        label="Confirm Password"
+        name="confirmPassword"
+        autocomplete="new-password"
+        saveOnInput
+        data-testid="ResetPasswordPage_confirmPassword"
+      />
+
+      {#if error}
+        <ErrorInfo message={error} />
+      {/if}
+
+      <div class="submit">
+        <FormSubmit
+          value={isSubmitting ? 'Resetting...' : 'Reset Password'}
+          disabled={isSubmitting || !token}
+          on:click={async e => {
+            error = null;
+            
+            if (!e.detail.newPassword || !e.detail.confirmPassword) {
+              error = 'Please fill in all fields';
+              return;
+            }
+            
+            if (e.detail.newPassword !== e.detail.confirmPassword) {
+              error = 'Passwords do not match';
+              return;
+            }
+            
+            if (e.detail.newPassword.length < 6) {
+              error = 'Password must be at least 6 characters long';
+              return;
+            }
+            
+            isSubmitting = true;
+            const resp = await apiCall('storage/reset-password', {
+              token,
+              newPassword: e.detail.newPassword,
+            });
+            isSubmitting = false;
+            
+            if (resp?.error) {
+              error = resp.error;
+              return;
+            }
+            
+            success = true;
+          }}
+          data-testid="ResetPasswordPage_submit"
+        />
+      </div>
+
+      <div class="back-link">
+        <Link internalRedirect="/login.html" data-testid="ResetPasswordPage_backToLogin">
+          Back to Login
+        </Link>
+      </div>
+    {/if}
+  </FormProviderCore>
+</SpecialPageLayout>
+
+<style>
+  .heading {
+    text-align: center;
+    margin: 1em;
+    font-size: xx-large;
+  }
+
+  .submit {
+    margin: var(--dim-large-form-margin);
+    display: flex;
+  }
+
+  .submit :global(input) {
+    flex: 1;
+    font-size: larger;
+  }
+
+  .submit :global(input:disabled) {
+    opacity: 0.6;
+    cursor: not-allowed;
+  }
+
+  .text {
+    margin-left: var(--dim-large-form-margin);
+    margin-right: var(--dim-large-form-margin);
+    margin-bottom: var(--dim-large-form-margin);
+  }
+
+  .success-message {
+    margin: var(--dim-large-form-margin);
+    padding: 15px;
+    background-color: var(--theme-bg-green);
+    border: 1px solid var(--theme-border);
+    border-radius: 4px;
+    color: var(--theme-font-1);
+  }
+
+  .back-link {
+    margin: var(--dim-large-form-margin);
+    text-align: center;
+  }
+</style>

packages/web/src/Screen.svelte

@@ -2,8 +2,6 @@
   import WidgetContainer from './widgets/WidgetContainer.svelte';
   import WidgetIconPanel from './widgets/WidgetIconPanel.svelte';
   import {
-    currentTheme,
-    currentThemeDefinition,
     isFileDragActive,
     leftPanelWidth,
     openedSnackbars,
@@ -11,13 +9,10 @@
     visibleWidgetSideBar,
     visibleCommandPalette,
     visibleTitleBar,
-    visibleToolbar,
-    systemThemeStore,
+    rightPanelWidget,
+    rightPanelWidth,
   } from './stores';
-  import TabsPanel from './tabpanel/TabsPanel.svelte';
-  import TabRegister from './tabpanel/TabRegister.svelte';
   import CommandPalette from './commands/CommandPalette.svelte';
-  import Toolbar from './widgets/Toolbar.svelte';
   import splitterDrag from './utility/splitterDrag';
   import CurrentDropDownMenu from './modals/CurrentDropDownMenu.svelte';
   import StatusBar from './widgets/StatusBar.svelte';
@@ -28,22 +23,15 @@
   import TitleBar from './widgets/TitleBar.svelte';
   import FontIcon from './icons/FontIcon.svelte';
   import getElectron from './utility/getElectron';
-  import TabsContainer from './tabpanel/TabsContainer.svelte';
   import MultiTabsContainer from './tabpanel/MultiTabsContainer.svelte';
+  import { currentThemeType } from './plugins/themes';
+  import RightWidgetContainer from './widgets/RightWidgetContainer.svelte';
 
-  $: currentThemeType = $currentThemeDefinition?.themeType == 'dark' ? 'theme-type-dark' : 'theme-type-light';
-
-  $: themeStyle = `<st` + `yle id="themePlugin">${$currentThemeDefinition?.themeCss}</st` + `yle>`;
+  $: currentThemeTypeClass = $currentThemeType == 'dark' ? 'theme-type-dark' : 'theme-type-light';
 
   const isElectron = !!getElectron();
 </script>
 
-<svelte:head>
-  {#if $currentThemeDefinition?.themeCss}
-    {@html themeStyle}
-  {/if}
-</svelte:head>
-
 <div class="not-supported" class:isElectron>
   <div class="m-5 big-icon"><FontIcon icon="img warn" /></div>
   <div class="m-3">Sorry, DbGate is not supported on mobile devices.</div>
@@ -51,7 +39,7 @@
 </div>
 
 <div
-  class={`${$currentTheme ?? $systemThemeStore} ${currentThemeType} root dbgate-screen`}
+  class={`${currentThemeTypeClass} root dbgate-screen`}
   class:isElectron
   use:dragDropFileTarget
   on:contextmenu={e => e.preventDefault()}
@@ -77,21 +65,28 @@
   </div>
   {#if $selectedWidget && $visibleWidgetSideBar}
     <div
-      class="horizontal-split-handle splitter"
+      class="horizontal-split-handle left-splitter"
       use:splitterDrag={'clientX'}
       on:resizeSplitter={e => leftPanelWidth.update(x => x + e.detail)}
     />
   {/if}
+  {#if $rightPanelWidget}
+    <div
+      class="horizontal-split-handle right-splitter"
+      use:splitterDrag={'clientX'}
+      on:resizeSplitter={e => rightPanelWidth.update(x => x - e.detail)}
+    />
+  {/if}
+  {#if $rightPanelWidget}
+    <div class="rightpanel">
+      <RightWidgetContainer />
+    </div>
+  {/if}
   {#if $visibleCommandPalette}
     <div class="commads">
       <CommandPalette />
     </div>
   {/if}
-  {#if $visibleToolbar}
-    <div class="toolbar">
-      <Toolbar />
-    </div>
-  {/if}
   <CurrentDropDownMenu />
   <ModalLayer />
   {#if $isFileDragActive}
@@ -106,7 +101,7 @@
 
 <style>
   .root {
-    color: var(--theme-font-1);
+    color: var(--theme-generic-font);
   }
   .iconbar {
     position: fixed;
@@ -115,11 +110,11 @@
     top: var(--dim-header-top);
     bottom: var(--dim-statusbar-height);
     width: var(--dim-widget-icon-size);
-    background: var(--theme-bg-inv-1);
+    background: var(--theme-widget-panel-background);
   }
   .statusbar {
     position: fixed;
-    background: var(--theme-bg-statusbar-inv);
+    background: var(--theme-statusbar-background);
     height: var(--dim-statusbar-height);
     left: 0;
     right: 0;
@@ -132,8 +127,22 @@
     left: var(--dim-widget-icon-size);
     bottom: var(--dim-statusbar-height);
     width: var(--dim-left-panel-width);
-    background-color: var(--theme-bg-1);
+    background-color: var(--theme-sidebar-background);
+    color: var(--theme-sidebar-foreground);
     display: flex;
+    border-right: var(--theme-sidebar-border);
+  }
+
+  .rightpanel {
+    position: fixed;
+    top: var(--dim-header-top);
+    right: 0;
+    bottom: var(--dim-statusbar-height);
+    width: var(--dim-right-panel-width);
+    background-color: var(--theme-altsidebar-background);
+    color: var(--theme-altsidebar-foreground);
+    display: flex;
+    border-left: var(--theme-altsidebar-border);
   }
   .commads {
     position: fixed;
@@ -146,16 +155,23 @@
     height: var(--dim-toolbar-height);
     left: 0;
     right: 0;
-    background: var(--theme-bg-1);
+    background: var(--theme-toolstrip-background);
   }
 
-  .splitter {
+  .left-splitter {
     position: absolute;
     top: var(--dim-header-top);
     bottom: var(--dim-statusbar-height);
     left: calc(var(--dim-widget-icon-size) + var(--dim-left-panel-width));
   }
 
+  .right-splitter {
+    position: absolute;
+    top: var(--dim-header-top);
+    bottom: var(--dim-statusbar-height);
+    right: var(--dim-content-right);
+  }
+
   .snackbar-container {
     z-index: 1000;
     position: fixed;
@@ -197,7 +213,7 @@
     top: var(--dim-header-top);
     left: var(--dim-content-left);
     bottom: var(--dim-statusbar-height);
-    right: 0;
-    background-color: var(--theme-bg-1);
+    right: var(--dim-content-right);
+    background-color: var(--theme-content-background);
   }
 </style>

packages/web/src/admin/FolderPermissionChooser.svelte

@@ -7,6 +7,7 @@
   export let onSetPermission;
   export let label;
   export let folder;
+  export let disabled = false;
 </script>
 
 <PermissionCheckBox
@@ -15,6 +16,7 @@
   permissions={$values.permissions}
   basePermissions={$values.basePermissions}
   {onSetPermission}
+  {disabled}
 />
 
 <div class="ml-4">
@@ -24,6 +26,7 @@
     permissions={$values.permissions}
     basePermissions={$values.basePermissions}
     {onSetPermission}
+    {disabled}
   />
   <PermissionCheckBox
     label="Write"
@@ -31,5 +34,6 @@
     permissions={$values.permissions}
     basePermissions={$values.basePermissions}
     {onSetPermission}
+    {disabled}
   />
 </div>

packages/web/src/ai/ThemeAiAssistantWidget.svelte

@@ -0,0 +1 @@
+This component is only for Premium edition