v6.6.0

feat: add useServerDatabaseFile for firebird

.github/workflows/build-app-pro-beta.yaml

@@ -39,7 +39,7 @@ jobs:
           repository: dbgate/dbgate-pro
           token: ${{ secrets.GH_TOKEN }}
           path: dbgate-pro
-          ref: 06d9017311c30974d5b4f03fe5fb9dbdabd29e31
+          ref: 36b6ce878c3c0a0c9623163c8a8b3bdeefc7da53
       - name: Merge dbgate/dbgate-pro
         run: |
           mkdir ../dbgate-pro

.github/workflows/build-app-pro.yaml

@@ -39,7 +39,7 @@ jobs:
           repository: dbgate/dbgate-pro
           token: ${{ secrets.GH_TOKEN }}
           path: dbgate-pro
-          ref: 06d9017311c30974d5b4f03fe5fb9dbdabd29e31
+          ref: 36b6ce878c3c0a0c9623163c8a8b3bdeefc7da53
       - name: Merge dbgate/dbgate-pro
         run: |
           mkdir ../dbgate-pro

.github/workflows/build-cloud-pro.yaml

@@ -39,7 +39,7 @@ jobs:
           repository: dbgate/dbgate-pro
           token: ${{ secrets.GH_TOKEN }}
           path: dbgate-pro
-          ref: 06d9017311c30974d5b4f03fe5fb9dbdabd29e31
+          ref: 36b6ce878c3c0a0c9623163c8a8b3bdeefc7da53
       - name: Merge dbgate/dbgate-pro
         run: |
           mkdir ../dbgate-pro

.github/workflows/build-docker-pro.yaml

@@ -44,7 +44,7 @@ jobs:
           repository: dbgate/dbgate-pro
           token: ${{ secrets.GH_TOKEN }}
           path: dbgate-pro
-          ref: 06d9017311c30974d5b4f03fe5fb9dbdabd29e31
+          ref: 36b6ce878c3c0a0c9623163c8a8b3bdeefc7da53
       - name: Merge dbgate/dbgate-pro
         run: |
           mkdir ../dbgate-pro

.github/workflows/build-npm-pro.yaml

@@ -32,7 +32,7 @@ jobs:
           repository: dbgate/dbgate-pro
           token: ${{ secrets.GH_TOKEN }}
           path: dbgate-pro
-          ref: 06d9017311c30974d5b4f03fe5fb9dbdabd29e31
+          ref: 36b6ce878c3c0a0c9623163c8a8b3bdeefc7da53
       - name: Merge dbgate/dbgate-pro
         run: |
           mkdir ../dbgate-pro
@@ -98,3 +98,8 @@ jobs:
           cd ..
           cd dbgate-merged/plugins/dbgate-plugin-cosmosdb
           npm publish
+      - name: Publish dbgate-plugin-firestore
+        run: |
+          cd ..
+          cd dbgate-merged/plugins/dbgate-plugin-firestore
+          npm publish

.github/workflows/e2e-pro.yaml

@@ -26,7 +26,7 @@ jobs:
           repository: dbgate/dbgate-pro
           token: ${{ secrets.GH_TOKEN }}
           path: dbgate-pro
-          ref: 06d9017311c30974d5b4f03fe5fb9dbdabd29e31
+          ref: 36b6ce878c3c0a0c9623163c8a8b3bdeefc7da53
       - name: Merge dbgate/dbgate-pro
         run: |
           mkdir ../dbgate-pro

.vscode/launch.json

@@ -1,20 +1,41 @@
 {
-    // Use IntelliSense to learn about possible attributes.
-    // Hover to view descriptions of existing attributes.
-    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
-    "version": "0.2.0",
-    "configurations": [
-        {
-            "type": "node",
-            "request": "launch",
-            "name": "Launch API",
-            "skipFiles": [
-                "<node_internals>/**"
-            ],
-            "program": "${workspaceFolder}/packages/api/src/index.js",
-            "outFiles": [
-                "${workspaceFolder}/**/*.js"
-            ]
-        }
-    ]
-}
+  "version": "0.2.0",
+  "configurations": [
+    {
+      "name": "Debug App",
+      "type": "node",
+      "request": "launch",
+      "program": "${workspaceFolder}/packages/api/src/index.js",
+      "envFile": "${workspaceFolder}/packages/api/.env",
+      "args": ["--listen-api"],
+      "console": "integratedTerminal",
+      "restart": true,
+      "runtimeExecutable": "node",
+      "skipFiles": ["<node_internals>/**"]
+    },
+    {
+      "name": "Debug App (Break on Start)",
+      "type": "node",
+      "request": "launch",
+      "program": "${workspaceFolder}/packages/api/src/index.js",
+      "args": ["--listen-api"],
+      "envFile": "${workspaceFolder}/.env",
+      "console": "integratedTerminal",
+      "restart": true,
+      "runtimeExecutable": "node",
+      "skipFiles": ["<node_internals>/**"],
+      "stopOnEntry": true
+    },
+    {
+      "name": "Attach to Process",
+      "type": "node",
+      "request": "attach",
+      "port": 9229,
+      "restart": true,
+      "localRoot": "${workspaceFolder}",
+      "remoteRoot": "${workspaceFolder}",
+      "skipFiles": ["<node_internals>/**"]
+    }
+  ]
+}
+

CHANGELOG.md

@@ -8,6 +8,42 @@ Builds:
  - linux - application for linux
  - win - application for Windows
 
+## 6.5.6
+- ADDED: New object window - quick access to most common functions
+- ADDED: Possibility to disable split query by empty line #1162
+- ADDED: Possibility to opt out authentication #1152
+- FIXED: Separate schema mode now works in Team Premium edition
+- FIXED: Handled situation, when user enters expired license, which is already prolonged
+- FIXED: Fixed some minor problems of charts
+
+## 6.5.5
+- ADDED: Administer cloud folder window
+- CHANGED: Cloud menu redesign
+- ADDED: Audit log (for Team Premium edition)
+- ADDED: Added new timeline chart type (line chart with time axis)
+- ADDED: Chart grouping (more measure determined from data)
+- CHANGED: Improved chart autodetection - string X axis (with bar type), COUNT as measure, split different measures
+- ADDED: Added chart data type detection
+- FIXED: Fixed chart displaying problems
+- FIXED: Fixed exporting chart to HTML
+- CHANGED: Choose COUNT measure without selecting underlying ID field (use virtual __count)
+- FIXED: Problems with authentification administration, especially for Postgres storage
+- CHANGED: Anonymous autentification (in Team Premium) is now by default disabled 
+
+## 6.5.3
+- CHANGED: Improved DbGate Cloud sign-in workflow
+- FIXED: Some fixes and error handling in new charts engine
+- ADDED: Charts - ability to choose aggregate function
+- CHANGED: Improved About window
+
+## 6.5.2
+- CHANGED: Autodetecting charts is disabled by default #1145
+- CHANGED: Improved chart displaying workflow
+- ADDED: Ability to close chart
+
+## 6.5.1
+- FIXED: DbGate Cloud e-mail sign-in method for desktop clients
+
 ## 6.5.0
 - ADDED: DbGate cloud - online storage for connections, SQL scripts and other objects
 - ADDED: Public knowledge base - common SQL scripts for specific DB engines (table sizes, index stats etc.)

README.md

@@ -38,6 +38,7 @@ DbGate is licensed under GPL-3.0 license and is free to use for any purpose.
 * Apache Cassandra
 * libSQL/Turso (Premium)
 * DuckDB
+* Firebird
 
 
 <a href="https://raw.githubusercontent.com/dbgate/dbgate/master/img/screenshot1.png">
@@ -88,10 +89,12 @@ DbGate is licensed under GPL-3.0 license and is free to use for any purpose.
 Any contributions are welcome. If you want to contribute without coding, consider following:
 
 * Tell your friends about DbGate or share on social networks - when more people will use DbGate, it will grow to be better
+* Purchase a [DbGate Premium](https://dbgate.io/purchase/premium/) liocense
 * Write review on [Slant.co](https://www.slant.co/improve/options/41086/~dbgate-review) or [G2](https://www.g2.com/products/dbgate/reviews) 
 * Create issue, if you find problem in app, or you have idea to new feature. If issue already exists, you could leave comment on it, to prioritise most wanted issues
 * Create some tutorial video on [youtube](https://www.youtube.com/playlist?list=PLCo7KjCVXhr0RfUSjM9wJMsp_ShL1q61A)
 * Become a backer on [GitHub sponsors](https://github.com/sponsors/dbgate) or [Open collective](https://opencollective.com/dbgate)
+* Add a SQL script to [Public Knowledge Base](https://github.com/dbgate/dbgate-knowledge-base)
 * Where a small coding is acceptable for you, you could [create plugin](https://docs.dbgate.io/plugin-development). Plugins for new themes can be created actually without JS coding
 
 Thank you!

e2e-tests/cypress.config.js

@@ -7,7 +7,9 @@ const path = require('path');
 
 module.exports = defineConfig({
   e2e: {
+    // baseUrl: 'http://localhost:3000',
     // trashAssetsBeforeRuns: false,
+    chromeWebSecurity: false,
 
     setupNodeEvents(on, config) {
       // implement node event listeners here
@@ -40,6 +42,12 @@ module.exports = defineConfig({
             case 'multi-sql':
               serverProcess = exec('yarn start:multi-sql');
               break;
+            case 'cloud':
+              serverProcess = exec('yarn start:cloud');
+              break;
+            case 'charts':
+              serverProcess = exec('yarn start:charts');
+              break;
           }
 
           await waitOn({ resources: ['http://localhost:3000'] });

e2e-tests/cypress/e2e/browse-data.cy.js

@@ -191,7 +191,8 @@ describe('Data browser data', () => {
   it('Query editor - join wizard', () => {
     cy.contains('MySql-connection').click();
     cy.contains('MyChinook').click();
-    cy.testid('TabsPanel_buttonNewQuery').click();
+    cy.testid('TabsPanel_buttonNewObject').click();
+    cy.testid('NewObjectModal_query').click();
     cy.wait(1000);
     cy.get('body').realType('select * from Invoice');
     cy.get('body').realPress('{enter}');
@@ -302,7 +303,8 @@ describe('Data browser data', () => {
   });
 
   it('Plugin tab', () => {
-    cy.testid('WidgetIconPanel_plugins').click();
+    cy.testid('WidgetIconPanel_settings').click();
+    cy.contains('Manage plugins').click();
     cy.contains('dbgate-plugin-theme-total-white').click();
     // text from plugin markdown
     cy.contains('Total white theme');
@@ -379,19 +381,25 @@ describe('Data browser data', () => {
     cy.themeshot('compare-database-settings');
   });
 
-  it('Query editor - AI assistant', () => {
+  it('Database chat', () => {
     cy.contains('MySql-connection').click();
     cy.contains('MyChinook').click();
-    cy.testid('TabsPanel_buttonNewQuery').click();
-    cy.testid('QueryTab_switchAiAssistantButton').click();
-    cy.testid('QueryAiAssistant_allowSendToAiServiceButton').click();
-    cy.testid('ConfirmModal_okButton').click();
-    cy.testid('QueryAiAssistant_promptInput').type('album names');
-    cy.testid('QueryAiAssistant_queryFromQuestionButton').click();
-    cy.contains('Use this', { timeout: 10000 }).click();
-    cy.testid('QueryTab_executeButton').click();
-    cy.contains('Balls to the Wall');
-    cy.themeshot('ai-assistant');
+    cy.testid('TabsPanel_buttonNewObject').click();
+    cy.testid('NewObjectModal_databaseChat').click();
+    cy.wait(1000);
+    cy.get('body').realType('find most popular artist');
+    cy.get('body').realPress('{enter}');
+    cy.testid('DatabaseChatTab_executeAllQueries', { timeout: 20000 }).click();
+    cy.wait(4000);
+    // cy.contains('Iron Maiden');
+    cy.themeshot('database-chat');
+
+    // cy.testid('DatabaseChatTab_promptInput').click();
+    // cy.get('body').realType('I need top 10 songs with the biggest income');
+    // cy.get('body').realPress('{enter}');
+    // cy.contains('Hot Girl', { timeout: 20000 });
+    // cy.wait(1000);
+    // cy.themeshot('database-chat');
   });
 
   it('Modify data', () => {
@@ -488,28 +496,4 @@ describe('Data browser data', () => {
     cy.testid('DataDeployTab_importIntoDb').click();
     cy.themeshot('data-replicator');
   });
-
-  it('Public Knowledge base - show chart', () => {
-    cy.contains('MySql-connection').click();
-    cy.contains('MyChinook').click();
-    cy.testid('WidgetIconPanel_cloud-public').click();
-    cy.testid('public-cloud-file-tag-mysql/folder-MySQL/tag-premium/top-tables-row-count.sql').click();
-    cy.testid('chart-canvas').should($c => expect($c[0].toDataURL()).to.match(/^data:image\/png;base64/));
-    cy.themeshot('public-knowledge-base-tables-sizes');
-  });
-
-  it('Private cloud - sign in', () => {
-    cy.contains('MySql-connection').click();
-    cy.contains('MyChinook').click();
-    cy.contains('Invoice').rightclick();
-    cy.contains('SQL template').click();
-    cy.contains('SELECT').click();
-    cy.testid('QueryTab_executeButton').click();
-    cy.contains('Chart 1').click();
-    cy.testid('JslChart_customizeButton').click();
-    cy.testid('ChartDefinitionEditor_chartTypeSelect').select('Bar');
-    cy.testid('ChartDefinitionEditor_chartTypeSelect').select('Line');
-    cy.testid('chart-canvas').should($c => expect($c[0].toDataURL()).to.match(/^data:image\/png;base64/));
-    cy.themeshot('query-result-chart');
-  });
 });

e2e-tests/cypress/e2e/charts.cy.js

@@ -0,0 +1,112 @@
+Cypress.on('uncaught:exception', (err, runnable) => {
+  // if the error message matches the one about WorkerGlobalScope importScripts
+  if (err.message.includes("Failed to execute 'importScripts' on 'WorkerGlobalScope'")) {
+    // return false to let Cypress know we intentionally want to ignore this error
+    return false;
+  }
+  // otherwise let Cypress throw the error
+});
+
+beforeEach(() => {
+  cy.visit('http://localhost:3000');
+  cy.viewport(1250, 900);
+});
+
+describe('Charts', () => {
+  it('Auto detect chart', () => {
+    cy.contains('MySql-connection').click();
+    cy.contains('charts_sample').click();
+    cy.testid('WidgetIconPanel_file').click();
+    cy.contains('chart1').click();
+    cy.contains('department_name');
+    // cy.testid('QueryTab_executeButton').click();
+    // cy.testid('QueryTab_openChartButton').click();
+    cy.testid('chart-canvas').should($c => expect($c[0].toDataURL()).to.match(/^data:image\/png;base64/));
+    cy.themeshot('choose-detected-chart');
+  });
+
+  it('Two line charts', () => {
+    cy.contains('MySql-connection').click();
+    cy.contains('MyChinook').click();
+    cy.testid('TabsPanel_buttonNewObject').click();
+    cy.testid('NewObjectModal_query').click();
+    cy.wait(1000);
+    cy.get('body').realType('SELECT InvoiceDate, Total from Invoice');
+    cy.contains('Execute').click();
+    cy.contains('Open chart').click();
+    cy.testid('chart-canvas').should($c => expect($c[0].toDataURL()).to.match(/^data:image\/png;base64/));
+    cy.themeshot('two-line-charts');
+  });
+
+  it('Invoice naive autodetection', () => {
+    cy.contains('MySql-connection').click();
+    cy.contains('MyChinook').click();
+    cy.testid('TabsPanel_buttonNewObject').click();
+    cy.testid('NewObjectModal_query').click();
+    cy.wait(1000);
+    cy.get('body').realType('SELECT * from Invoice');
+    cy.contains('Execute').click();
+    cy.contains('Open chart').click();
+    cy.testid('chart-canvas').should($c => expect($c[0].toDataURL()).to.match(/^data:image\/png;base64/));
+    cy.themeshot('chart-naive-autodetection');
+  });
+
+  it('Invoice by country - grouped chart', () => {
+    cy.contains('MySql-connection').click();
+    cy.contains('MyChinook').click();
+    cy.testid('TabsPanel_buttonNewObject').click();
+    cy.testid('NewObjectModal_query').click();
+    cy.wait(1000);
+    cy.get('body').realType(
+      "SELECT InvoiceDate, Total, BillingCountry from Invoice where BillingCountry in ('USA', 'Canada', 'Brazil', 'France', 'Germany')"
+    );
+    cy.contains('Execute').click();
+    cy.contains('Open chart').click();
+    cy.testid('ChartSelector_chart_1').click();
+    cy.testid('JslChart_customizeButton').click();
+
+    cy.testid('chart-canvas').should($c => expect($c[0].toDataURL()).to.match(/^data:image\/png;base64/));
+    cy.themeshot('chart-grouped-autodetected');
+
+    cy.testid('ChartDefinitionEditor_chartTypeSelect').select('Bar');
+    cy.testid('ChartDefinitionEditor_xAxisTransformSelect').select('Date (Year)');
+
+    cy.testid('chart-canvas').should($c => expect($c[0].toDataURL()).to.match(/^data:image\/png;base64/));
+    cy.themeshot('chart-grouped-bars');
+  });
+
+  it('Public Knowledge base - show chart', () => {
+    cy.contains('MySql-connection').click();
+    cy.contains('MyChinook').click();
+    cy.testid('WidgetIconPanel_cloud-public').click();
+    cy.testid('public-cloud-file-tag-mysql/folder-MySQL/tag-premium/top-tables-row-count.sql').click();
+    cy.testid('chart-canvas').should($c => expect($c[0].toDataURL()).to.match(/^data:image\/png;base64/));
+    cy.themeshot('public-knowledge-base-tables-sizes');
+  });
+
+  it('Auto detect chart', () => {
+    cy.contains('MySql-connection').click();
+    cy.contains('MyChinook').click();
+    cy.contains('Invoice').rightclick();
+    cy.contains('SQL template').click();
+    cy.contains('SELECT').click();
+    cy.testid('QueryTab_detectChartButton').click();
+    cy.testid('QueryTab_executeButton').click();
+    cy.contains('Chart 1').click();
+    cy.testid('ChartSelector_chart_0').click();
+    cy.testid('JslChart_customizeButton').click();
+    cy.testid('ChartDefinitionEditor_chartTypeSelect').select('Bar');
+    cy.testid('ChartDefinitionEditor_chartTypeSelect').select('Line');
+    cy.testid('chart-canvas').should($c => expect($c[0].toDataURL()).to.match(/^data:image\/png;base64/));
+    cy.themeshot('query-result-chart');
+  });
+
+  it('New object window', () => {
+    cy.contains('MySql-connection').click();
+    cy.contains('MyChinook').click();
+    cy.contains('Invoice').click();
+    cy.testid('WidgetIconPanel_addButton').click();
+    cy.contains('Compare database');
+    cy.themeshot('new-object-window');
+  });
+});

e2e-tests/cypress/e2e/cloud.cy.js

@@ -0,0 +1,56 @@
+Cypress.on('uncaught:exception', (err, runnable) => {
+  // if the error message matches the one about WorkerGlobalScope importScripts
+  if (err.message.includes("Failed to execute 'importScripts' on 'WorkerGlobalScope'")) {
+    // return false to let Cypress know we intentionally want to ignore this error
+    return false;
+  }
+  // otherwise let Cypress throw the error
+});
+
+beforeEach(() => {
+  cy.visit('http://localhost:3000');
+  cy.viewport(1250, 900);
+});
+
+describe('Cloud tests', () => {
+  it('Private cloud', () => {
+    cy.testid('WidgetIconPanel_cloudAccount');
+    cy.window().then(win => {
+      win.__loginToCloudTest('dbgate.test@gmail.com');
+    });
+    cy.contains('dbgate.test@gmail.com');
+
+    // cy.testid('WidgetIconPanel_cloudAccount').click();
+
+    // cy.origin('https://identity.dbgate.io', () => {
+    //   cy.contains('Sign in with GitHub').click();
+    // });
+
+    // cy.origin('https://github.com', () => {
+    //   cy.get('#login_field').type('dbgatetest');
+    //   cy.get('#password').type('Pwd2020Db');
+    //   cy.get('input[type="submit"]').click();
+    // });
+
+    // cy.wait(3000);
+
+    // cy.location('origin').then(origin => {
+    //   if (origin === 'https://github.com') {
+    //     // Still on github.com → an authorization step is waiting
+    //     cy.origin('https://github.com', () => {
+    //       // Try once, don't wait the full default timeout
+    //       cy.get('button[data-octo-click="oauth_application_authorization"]', { timeout: 500, log: false }).click(); // if the button exists it will be clicked
+    //       // if not, the short timeout elapses and we drop out
+    //     });
+    //   } else {
+    //     // Already back on localhost – nothing to authorize
+    //     cy.log('OAuth redirect skipped the Authorize screen');
+    //   }
+    // });
+
+    cy.contains('Testing Connections').rightclick();
+    cy.contains('Administrate access').click();
+    cy.contains('User email');
+    cy.themeshot('administer-shared-folder');
+  });
+});

e2e-tests/cypress/e2e/multi-sql.cy.js

@@ -59,7 +59,8 @@ describe('Transactions', () => {
 
     cy.contains(connectionName).click();
     if (databaseName) cy.contains(databaseName).click();
-    cy.testid('TabsPanel_buttonNewQuery').click();
+    cy.testid('TabsPanel_buttonNewObject').click();
+    cy.testid('NewObjectModal_query').click();
     cy.wait(1000);
     cy.get('body').type(
       formatQueryWithoutParams(driver, "INSERT INTO ~categories (~category_id, ~category_name) VALUES (5, 'test');")

e2e-tests/cypress/e2e/team.cy.js

@@ -1,3 +1,12 @@
+Cypress.on('uncaught:exception', (err, runnable) => {
+  // if the error message matches the one about WorkerGlobalScope importScripts
+  if (err.message.includes("Failed to execute 'importScripts' on 'WorkerGlobalScope'")) {
+    // return false to let Cypress know we intentionally want to ignore this error
+    return false;
+  }
+  // otherwise let Cypress throw the error
+});
+
 beforeEach(() => {
   cy.visit('http://localhost:3000');
   cy.viewport(1250, 900);
@@ -80,4 +89,34 @@ describe('Team edition tests', () => {
     cy.testid('AdminMenuWidget_itemUsers').click();
     cy.contains('test@example.com');
   });
+
+  it('Audit logging', () => {
+    cy.testid('LoginPage_linkAdmin').click();
+    cy.testid('LoginPage_password').type('adminpwd');
+    cy.testid('LoginPage_submitLogin').click();
+
+    cy.testid('AdminMenuWidget_itemAuditLog').click();
+    cy.contains('Audit log is not enabled');
+    cy.testid('AdminMenuWidget_itemSettings').click();
+    cy.testid('AdminSettingsTab_auditLogCheckbox').click();
+    cy.testid('AdminMenuWidget_itemAuditLog').click();
+    cy.contains('No data for selected date');
+
+    cy.testid('AdminMenuWidget_itemConnections').click();
+    cy.contains('Open table').click();
+    cy.contains('displayName');
+    cy.get('.toolstrip').contains('Export').click();
+    cy.contains('CSV file').click();
+
+    cy.testid('AdminMenuWidget_itemUsers').click();
+    cy.contains('Open table').click();
+    cy.contains('login');
+    cy.get('.toolstrip').contains('Export').click();
+    cy.contains('XML file').click();
+
+    cy.testid('AdminMenuWidget_itemAuditLog').click();
+    cy.testid('AdminAuditLogTab_refreshButton').click();
+    cy.contains('Exporting query').click();
+    cy.themeshot('auditlog');
+  });
 });

e2e-tests/data/charts-sample/departments.jsonl

@@ -0,0 +1,6 @@
+{"__isStreamHeader":true,"pureName":"departments","schemaName":"dbo","objectId":1205579333,"createDate":"2025-06-12T10:30:34.083Z","modifyDate":"2025-06-12T10:30:34.120Z","contentHash":"2025-06-12T10:30:34.120Z","columns":[{"columnName":"id","dataType":"int","notNull":true,"autoIncrement":false,"defaultValue":null,"defaultConstraint":null,"computedExpression":null,"hasAutoValue":false},{"columnName":"name","dataType":"varchar(100)","notNull":true,"autoIncrement":false,"defaultValue":null,"defaultConstraint":null,"computedExpression":null,"hasAutoValue":false}],"primaryKey":{"constraintName":"PK__departme__3213E83FE8E7043D","schemaName":"dbo","pureName":"departments","constraintType":"primaryKey","columns":[{"columnName":"id"}]},"foreignKeys":[],"indexes":[],"uniques":[],"engine":"mssql@dbgate-plugin-mssql"}
+{"id":1,"name":"IT"}
+{"id":2,"name":"Marketing"}
+{"id":3,"name":"Finance"}
+{"id":4,"name":"Human Resources"}
+{"id":5,"name":"Research and Development"}

e2e-tests/data/charts-sample/departments.table.yaml

@@ -0,0 +1,12 @@
+name: departments
+columns:
+  - name: id
+    type: int
+    default: null
+    notNull: true
+  - name: name
+    type: varchar(100)
+    default: null
+    notNull: true
+primaryKey:
+  - id

e2e-tests/data/charts-sample/employee_project.jsonl

@@ -0,0 +1,39 @@
+{"__isStreamHeader":true,"pureName":"employee_project","schemaName":"dbo","objectId":1333579789,"createDate":"2025-06-12T10:30:34.133Z","modifyDate":"2025-06-12T10:30:34.133Z","contentHash":"2025-06-12T10:30:34.133Z","columns":[{"columnName":"employee_id","dataType":"int","notNull":true,"autoIncrement":false,"defaultValue":null,"defaultConstraint":null,"computedExpression":null,"hasAutoValue":false},{"columnName":"project_id","dataType":"int","notNull":true,"autoIncrement":false,"defaultValue":null,"defaultConstraint":null,"computedExpression":null,"hasAutoValue":false},{"columnName":"role","dataType":"varchar(50)","notNull":false,"autoIncrement":false,"defaultValue":null,"defaultConstraint":null,"computedExpression":null,"hasAutoValue":false}],"primaryKey":{"constraintName":"PK__employee__2EE9924949ED9668","schemaName":"dbo","pureName":"employee_project","constraintType":"primaryKey","columns":[{"columnName":"employee_id"},{"columnName":"project_id"}]},"foreignKeys":[{"constraintName":"FK__employee___emplo__5165187F","constraintType":"foreignKey","schemaName":"dbo","pureName":"employee_project","refSchemaName":"dbo","refTableName":"employees","updateAction":"NO ACTION","deleteAction":"NO ACTION","columns":[{"columnName":"employee_id","refColumnName":"id"}]},{"constraintName":"FK__employee___proje__52593CB8","constraintType":"foreignKey","schemaName":"dbo","pureName":"employee_project","refSchemaName":"dbo","refTableName":"projects","updateAction":"NO ACTION","deleteAction":"NO ACTION","columns":[{"columnName":"project_id","refColumnName":"id"}]}],"indexes":[],"uniques":[],"engine":"mssql@dbgate-plugin-mssql"}
+{"employee_id":1,"project_id":6,"role":"Manager"}
+{"employee_id":1,"project_id":8,"role":"Developer"}
+{"employee_id":2,"project_id":7,"role":"Tester"}
+{"employee_id":2,"project_id":8,"role":"Developer"}
+{"employee_id":3,"project_id":4,"role":"Analyst"}
+{"employee_id":3,"project_id":6,"role":"Developer"}
+{"employee_id":4,"project_id":2,"role":"Manager"}
+{"employee_id":4,"project_id":4,"role":"Analyst"}
+{"employee_id":4,"project_id":5,"role":"Analyst"}
+{"employee_id":5,"project_id":5,"role":"Tester"}
+{"employee_id":6,"project_id":1,"role":"Analyst"}
+{"employee_id":6,"project_id":6,"role":"Tester"}
+{"employee_id":6,"project_id":9,"role":"Manager"}
+{"employee_id":7,"project_id":8,"role":"Manager"}
+{"employee_id":8,"project_id":10,"role":"Analyst"}
+{"employee_id":9,"project_id":2,"role":"Analyst"}
+{"employee_id":9,"project_id":6,"role":"Analyst"}
+{"employee_id":9,"project_id":7,"role":"Developer"}
+{"employee_id":10,"project_id":2,"role":"Manager"}
+{"employee_id":10,"project_id":6,"role":"Analyst"}
+{"employee_id":11,"project_id":1,"role":"Tester"}
+{"employee_id":12,"project_id":4,"role":"Tester"}
+{"employee_id":13,"project_id":2,"role":"Developer"}
+{"employee_id":13,"project_id":3,"role":"Analyst"}
+{"employee_id":13,"project_id":7,"role":"Developer"}
+{"employee_id":14,"project_id":3,"role":"Developer"}
+{"employee_id":14,"project_id":9,"role":"Manager"}
+{"employee_id":15,"project_id":1,"role":"Developer"}
+{"employee_id":15,"project_id":5,"role":"Manager"}
+{"employee_id":16,"project_id":3,"role":"Tester"}
+{"employee_id":16,"project_id":5,"role":"Developer"}
+{"employee_id":17,"project_id":6,"role":"Analyst"}
+{"employee_id":18,"project_id":1,"role":"Tester"}
+{"employee_id":18,"project_id":5,"role":"Tester"}
+{"employee_id":18,"project_id":6,"role":"Manager"}
+{"employee_id":19,"project_id":6,"role":"Analyst"}
+{"employee_id":20,"project_id":2,"role":"Developer"}
+{"employee_id":20,"project_id":4,"role":"Developer"}

e2e-tests/data/charts-sample/employee_project.table.yaml

@@ -0,0 +1,18 @@
+name: employee_project
+columns:
+  - name: employee_id
+    type: int
+    default: null
+    notNull: true
+    references: employees
+  - name: project_id
+    type: int
+    default: null
+    notNull: true
+    references: projects
+  - name: role
+    type: varchar(50)
+    default: null
+primaryKey:
+  - employee_id
+  - project_id

e2e-tests/data/charts-sample/employees.jsonl

@@ -0,0 +1,21 @@
+{"__isStreamHeader":true,"pureName":"employees","schemaName":"dbo","objectId":1237579447,"createDate":"2025-06-12T10:30:34.113Z","modifyDate":"2025-06-12T12:35:22.140Z","contentHash":"2025-06-12T12:35:22.140Z","columns":[{"columnName":"id","dataType":"int","notNull":true,"autoIncrement":false,"defaultValue":null,"defaultConstraint":null,"computedExpression":null,"hasAutoValue":false},{"columnName":"name","dataType":"varchar(100)","notNull":true,"autoIncrement":false,"defaultValue":null,"defaultConstraint":null,"computedExpression":null,"hasAutoValue":false},{"columnName":"email","dataType":"varchar(100)","notNull":true,"autoIncrement":false,"defaultValue":null,"defaultConstraint":null,"computedExpression":null,"hasAutoValue":false},{"columnName":"hire_date","dataType":"date","notNull":true,"autoIncrement":false,"defaultValue":null,"defaultConstraint":null,"computedExpression":null,"hasAutoValue":false},{"columnName":"department_id","dataType":"int","notNull":false,"autoIncrement":false,"defaultValue":null,"defaultConstraint":null,"computedExpression":null,"hasAutoValue":false}],"primaryKey":{"constraintName":"PK__employee__3213E83FE576E55A","schemaName":"dbo","pureName":"employees","constraintType":"primaryKey","columns":[{"columnName":"id"}]},"foreignKeys":[{"constraintName":"FK__employees__depar__4CA06362","constraintType":"foreignKey","schemaName":"dbo","pureName":"employees","refSchemaName":"dbo","refTableName":"departments","updateAction":"NO ACTION","deleteAction":"NO ACTION","columns":[{"columnName":"department_id","refColumnName":"id"}]}],"indexes":[],"uniques":[{"constraintName":"UQ__employee__AB6E6164E18D883F","columns":[{"columnName":"email"}]}],"engine":"mssql@dbgate-plugin-mssql"}
+{"id":1,"name":"John Smith","email":"john.smith@example.com","hire_date":"2018-07-09T00:00:00.000Z","department_id":2}
+{"id":2,"name":"Jane Garcia","email":"jane.garcia@example.com","hire_date":"2019-10-13T00:00:00.000Z","department_id":5}
+{"id":3,"name":"Grace Smith","email":"grace.smith@example.com","hire_date":"2019-03-16T00:00:00.000Z","department_id":1}
+{"id":4,"name":"Charlie Williams","email":"charlie.williams@example.com","hire_date":"2020-10-18T00:00:00.000Z","department_id":2}
+{"id":5,"name":"Eve Brown","email":"eve.brown@example.com","hire_date":"2018-04-12T00:00:00.000Z","department_id":4}
+{"id":6,"name":"Alice Moore","email":"alice.moore@example.com","hire_date":"2019-04-20T00:00:00.000Z","department_id":2}
+{"id":7,"name":"Eve Williams","email":"eve.williams@example.com","hire_date":"2020-04-26T00:00:00.000Z","department_id":4}
+{"id":8,"name":"Eve Jones","email":"eve.jones@example.com","hire_date":"2022-10-04T00:00:00.000Z","department_id":3}
+{"id":9,"name":"Diana Miller","email":"diana.miller@example.com","hire_date":"2021-03-28T00:00:00.000Z","department_id":1}
+{"id":10,"name":"Diana Smith","email":"diana.smith@example.com","hire_date":"2018-04-12T00:00:00.000Z","department_id":2}
+{"id":11,"name":"Hank Johnson","email":"hank.johnson@example.com","hire_date":"2020-09-16T00:00:00.000Z","department_id":2}
+{"id":12,"name":"Frank Miller","email":"frank.miller@example.com","hire_date":"2023-01-12T00:00:00.000Z","department_id":4}
+{"id":13,"name":"Jane Brown","email":"jane.brown@example.com","hire_date":"2023-05-07T00:00:00.000Z","department_id":3}
+{"id":14,"name":"Grace Davis","email":"grace.davis@example.com","hire_date":"2019-08-22T00:00:00.000Z","department_id":3}
+{"id":15,"name":"Jane Black","email":"jane.black@example.com","hire_date":"2019-04-28T00:00:00.000Z","department_id":2}
+{"id":16,"name":"Charlie Smith","email":"charlie.smith@example.com","hire_date":"2019-06-12T00:00:00.000Z","department_id":5}
+{"id":17,"name":"Eve Johnson","email":"eve.johnson@example.com","hire_date":"2020-11-07T00:00:00.000Z","department_id":5}
+{"id":18,"name":"Jane Johnson","email":"jane.johnson@example.com","hire_date":"2020-04-06T00:00:00.000Z","department_id":2}
+{"id":19,"name":"Hank Brown","email":"hank.brown@example.com","hire_date":"2023-05-10T00:00:00.000Z","department_id":2}
+{"id":20,"name":"Frank Jones","email":"frank.jones@example.com","hire_date":"2020-10-26T00:00:00.000Z","department_id":1}

e2e-tests/data/charts-sample/employees.table.yaml

@@ -0,0 +1,28 @@
+name: employees
+columns:
+  - name: id
+    type: int
+    default: null
+    notNull: true
+  - name: name
+    type: varchar(100)
+    default: null
+    notNull: true
+  - name: email
+    type: varchar(100)
+    default: null
+    notNull: true
+  - name: hire_date
+    type: date
+    default: null
+    notNull: true
+  - name: department_id
+    type: int
+    default: null
+    references: departments
+primaryKey:
+  - id
+uniques:
+  - name: UQ__employee__AB6E6164E18D883F
+    columns:
+      - email

e2e-tests/data/charts-sample/finance_reports.jsonl

@@ -0,0 +1,141 @@
+{"__isStreamHeader":true,"pureName":"finance_reports","schemaName":"dbo","objectId":338100245,"createDate":"2025-06-23T12:15:08.727Z","modifyDate":"2025-06-23T12:15:08.750Z","contentHash":"2025-06-23T12:15:08.750Z","columns":[{"columnName":"id","dataType":"int","notNull":true,"autoIncrement":false,"defaultValue":null,"defaultConstraint":null,"computedExpression":null,"hasAutoValue":false},{"columnName":"date","dataType":"date","notNull":true,"autoIncrement":false,"defaultValue":null,"defaultConstraint":null,"computedExpression":null,"hasAutoValue":false},{"columnName":"profit","dataType":"money","notNull":true,"autoIncrement":false,"defaultValue":null,"defaultConstraint":null,"computedExpression":null,"hasAutoValue":false}],"foreignKeys":[{"constraintName":"project_id","constraintType":"foreignKey","schemaName":"dbo","pureName":"finance_reports","refSchemaName":"dbo","refTableName":"projects","updateAction":"NO ACTION","deleteAction":"NO ACTION","columns":[{"columnName":"id","refColumnName":"id"}]}],"indexes":[],"uniques":[],"engine":"mssql@dbgate-plugin-mssql"}
+{"id":1,"date":"2022-01-01T00:00:00.000Z","profit":73923.4}
+{"id":1,"date":"2022-01-31T00:00:00.000Z","profit":21837.75}
+{"id":1,"date":"2022-03-02T00:00:00.000Z","profit":67859.8}
+{"id":1,"date":"2022-04-01T00:00:00.000Z","profit":77403.3}
+{"id":1,"date":"2022-05-01T00:00:00.000Z","profit":84083.19}
+{"id":1,"date":"2022-05-31T00:00:00.000Z","profit":30040.55}
+{"id":1,"date":"2022-06-30T00:00:00.000Z","profit":50947.14}
+{"id":1,"date":"2022-07-30T00:00:00.000Z","profit":63345.62}
+{"id":1,"date":"2022-08-29T00:00:00.000Z","profit":23819.45}
+{"id":1,"date":"2022-09-28T00:00:00.000Z","profit":-25919.19}
+{"id":1,"date":"2022-10-28T00:00:00.000Z","profit":27967.6}
+{"id":1,"date":"2022-11-27T00:00:00.000Z","profit":-37402.36}
+{"id":1,"date":"2022-12-27T00:00:00.000Z","profit":94528.8}
+{"id":1,"date":"2023-01-26T00:00:00.000Z","profit":29491.03}
+{"id":1,"date":"2023-02-25T00:00:00.000Z","profit":81541.29}
+{"id":2,"date":"2022-01-01T00:00:00.000Z","profit":18070.94}
+{"id":2,"date":"2022-01-31T00:00:00.000Z","profit":-40609.87}
+{"id":2,"date":"2022-03-02T00:00:00.000Z","profit":42435.51}
+{"id":2,"date":"2022-04-01T00:00:00.000Z","profit":-11915.15}
+{"id":2,"date":"2022-05-01T00:00:00.000Z","profit":-37417.4}
+{"id":2,"date":"2022-05-31T00:00:00.000Z","profit":23028.66}
+{"id":2,"date":"2022-06-30T00:00:00.000Z","profit":-6895.49}
+{"id":2,"date":"2022-07-30T00:00:00.000Z","profit":63114.54}
+{"id":2,"date":"2022-08-29T00:00:00.000Z","profit":94646.99}
+{"id":2,"date":"2022-09-28T00:00:00.000Z","profit":99560.77}
+{"id":2,"date":"2022-10-28T00:00:00.000Z","profit":62216.22}
+{"id":2,"date":"2022-11-27T00:00:00.000Z","profit":85094.32}
+{"id":2,"date":"2022-12-27T00:00:00.000Z","profit":-23378.37}
+{"id":2,"date":"2023-01-26T00:00:00.000Z","profit":47635.86}
+{"id":2,"date":"2023-02-25T00:00:00.000Z","profit":33727.72}
+{"id":3,"date":"2022-01-01T00:00:00.000Z","profit":33088.03}
+{"id":3,"date":"2022-01-31T00:00:00.000Z","profit":66668.91}
+{"id":3,"date":"2022-03-02T00:00:00.000Z","profit":5344.27}
+{"id":3,"date":"2022-04-01T00:00:00.000Z","profit":22122.99}
+{"id":3,"date":"2022-05-01T00:00:00.000Z","profit":27342.01}
+{"id":3,"date":"2022-05-31T00:00:00.000Z","profit":55479.42}
+{"id":3,"date":"2022-06-30T00:00:00.000Z","profit":35956.11}
+{"id":3,"date":"2022-07-30T00:00:00.000Z","profit":9667.12}
+{"id":3,"date":"2022-08-29T00:00:00.000Z","profit":63430.18}
+{"id":3,"date":"2022-09-28T00:00:00.000Z","profit":-4883.41}
+{"id":3,"date":"2022-10-28T00:00:00.000Z","profit":38902.8}
+{"id":3,"date":"2022-11-27T00:00:00.000Z","profit":-25500.13}
+{"id":3,"date":"2022-12-27T00:00:00.000Z","profit":65074.21}
+{"id":3,"date":"2023-01-26T00:00:00.000Z","profit":12570.27}
+{"id":3,"date":"2023-02-25T00:00:00.000Z","profit":35418.36}
+{"id":4,"date":"2022-01-01T00:00:00.000Z","profit":68282.98}
+{"id":4,"date":"2022-01-31T00:00:00.000Z","profit":77778.99}
+{"id":4,"date":"2022-03-02T00:00:00.000Z","profit":95490.49}
+{"id":4,"date":"2022-04-01T00:00:00.000Z","profit":-44466.37}
+{"id":4,"date":"2022-05-01T00:00:00.000Z","profit":40215.71}
+{"id":4,"date":"2022-05-31T00:00:00.000Z","profit":-31228.87}
+{"id":4,"date":"2022-06-30T00:00:00.000Z","profit":60667.69}
+{"id":4,"date":"2022-07-30T00:00:00.000Z","profit":71439.16}
+{"id":4,"date":"2022-08-29T00:00:00.000Z","profit":-25077.4}
+{"id":4,"date":"2022-09-28T00:00:00.000Z","profit":-36128.2}
+{"id":4,"date":"2022-10-28T00:00:00.000Z","profit":36727.68}
+{"id":4,"date":"2022-11-27T00:00:00.000Z","profit":-24207.2}
+{"id":4,"date":"2022-12-27T00:00:00.000Z","profit":63846.96}
+{"id":5,"date":"2022-01-01T00:00:00.000Z","profit":21648.3}
+{"id":5,"date":"2022-01-31T00:00:00.000Z","profit":59263.22}
+{"id":5,"date":"2022-03-02T00:00:00.000Z","profit":49154.51}
+{"id":5,"date":"2022-04-01T00:00:00.000Z","profit":34787.48}
+{"id":5,"date":"2022-05-01T00:00:00.000Z","profit":-24120.19}
+{"id":5,"date":"2022-05-31T00:00:00.000Z","profit":98437.86}
+{"id":5,"date":"2022-06-30T00:00:00.000Z","profit":18614.77}
+{"id":5,"date":"2022-07-30T00:00:00.000Z","profit":17680.34}
+{"id":5,"date":"2022-08-29T00:00:00.000Z","profit":74406.86}
+{"id":5,"date":"2022-09-28T00:00:00.000Z","profit":61845.3}
+{"id":5,"date":"2022-10-28T00:00:00.000Z","profit":-37889.59}
+{"id":5,"date":"2022-11-27T00:00:00.000Z","profit":76651.05}
+{"id":5,"date":"2022-12-27T00:00:00.000Z","profit":58739.6}
+{"id":5,"date":"2023-01-26T00:00:00.000Z","profit":82605.85}
+{"id":6,"date":"2022-01-01T00:00:00.000Z","profit":-5206.8}
+{"id":6,"date":"2022-01-31T00:00:00.000Z","profit":27498.27}
+{"id":6,"date":"2022-03-02T00:00:00.000Z","profit":-2939.84}
+{"id":6,"date":"2022-04-01T00:00:00.000Z","profit":-37261.08}
+{"id":6,"date":"2022-05-01T00:00:00.000Z","profit":37069.04}
+{"id":6,"date":"2022-05-31T00:00:00.000Z","profit":524.88}
+{"id":6,"date":"2022-06-30T00:00:00.000Z","profit":-29620.85}
+{"id":6,"date":"2022-07-30T00:00:00.000Z","profit":35540.81}
+{"id":6,"date":"2022-08-29T00:00:00.000Z","profit":20608.94}
+{"id":6,"date":"2022-09-28T00:00:00.000Z","profit":34809.33}
+{"id":6,"date":"2022-10-28T00:00:00.000Z","profit":-44949.05}
+{"id":6,"date":"2022-11-27T00:00:00.000Z","profit":-22524.26}
+{"id":6,"date":"2022-12-27T00:00:00.000Z","profit":37841.58}
+{"id":7,"date":"2022-01-01T00:00:00.000Z","profit":6903.17}
+{"id":7,"date":"2022-01-31T00:00:00.000Z","profit":58480.84}
+{"id":7,"date":"2022-03-02T00:00:00.000Z","profit":48217.34}
+{"id":7,"date":"2022-04-01T00:00:00.000Z","profit":73592.44}
+{"id":7,"date":"2022-05-01T00:00:00.000Z","profit":-21831.18}
+{"id":7,"date":"2022-05-31T00:00:00.000Z","profit":-40926.16}
+{"id":7,"date":"2022-06-30T00:00:00.000Z","profit":62299.5}
+{"id":7,"date":"2022-07-30T00:00:00.000Z","profit":95376.53}
+{"id":7,"date":"2022-08-29T00:00:00.000Z","profit":-13317.36}
+{"id":7,"date":"2022-09-28T00:00:00.000Z","profit":81565.05}
+{"id":7,"date":"2022-10-28T00:00:00.000Z","profit":77420.52}
+{"id":7,"date":"2022-11-27T00:00:00.000Z","profit":-12052.47}
+{"id":7,"date":"2022-12-27T00:00:00.000Z","profit":37742.07}
+{"id":7,"date":"2023-01-26T00:00:00.000Z","profit":-8057.99}
+{"id":8,"date":"2022-01-01T00:00:00.000Z","profit":27213.73}
+{"id":8,"date":"2022-01-31T00:00:00.000Z","profit":34271.75}
+{"id":8,"date":"2022-03-02T00:00:00.000Z","profit":-44549.47}
+{"id":8,"date":"2022-04-01T00:00:00.000Z","profit":15236.34}
+{"id":8,"date":"2022-05-01T00:00:00.000Z","profit":-27759.81}
+{"id":8,"date":"2022-05-31T00:00:00.000Z","profit":7955.12}
+{"id":8,"date":"2022-06-30T00:00:00.000Z","profit":-34484.38}
+{"id":8,"date":"2022-07-30T00:00:00.000Z","profit":-49758.7}
+{"id":8,"date":"2022-08-29T00:00:00.000Z","profit":-41990.86}
+{"id":8,"date":"2022-09-28T00:00:00.000Z","profit":58123.01}
+{"id":8,"date":"2022-10-28T00:00:00.000Z","profit":30128.78}
+{"id":8,"date":"2022-11-27T00:00:00.000Z","profit":-10151.17}
+{"id":8,"date":"2022-12-27T00:00:00.000Z","profit":54048.33}
+{"id":8,"date":"2023-01-26T00:00:00.000Z","profit":-43123.17}
+{"id":9,"date":"2022-01-01T00:00:00.000Z","profit":61031.83}
+{"id":9,"date":"2022-01-31T00:00:00.000Z","profit":68577.58}
+{"id":9,"date":"2022-03-02T00:00:00.000Z","profit":88698.97}
+{"id":9,"date":"2022-04-01T00:00:00.000Z","profit":8906.03}
+{"id":9,"date":"2022-05-01T00:00:00.000Z","profit":28824.73}
+{"id":9,"date":"2022-05-31T00:00:00.000Z","profit":88280.34}
+{"id":9,"date":"2022-06-30T00:00:00.000Z","profit":35266.09}
+{"id":9,"date":"2022-07-30T00:00:00.000Z","profit":-38025.36}
+{"id":9,"date":"2022-08-29T00:00:00.000Z","profit":-12118.53}
+{"id":9,"date":"2022-09-28T00:00:00.000Z","profit":-27265.86}
+{"id":9,"date":"2022-10-28T00:00:00.000Z","profit":56870.57}
+{"id":9,"date":"2022-11-27T00:00:00.000Z","profit":88078.95}
+{"id":9,"date":"2022-12-27T00:00:00.000Z","profit":-24059.67}
+{"id":9,"date":"2023-01-26T00:00:00.000Z","profit":-13301.43}
+{"id":10,"date":"2022-01-01T00:00:00.000Z","profit":-22479.23}
+{"id":10,"date":"2022-01-31T00:00:00.000Z","profit":8106.27}
+{"id":10,"date":"2022-03-02T00:00:00.000Z","profit":69372.19}
+{"id":10,"date":"2022-04-01T00:00:00.000Z","profit":-11895.74}
+{"id":10,"date":"2022-05-01T00:00:00.000Z","profit":-33206.5}
+{"id":10,"date":"2022-05-31T00:00:00.000Z","profit":56073.34}
+{"id":10,"date":"2022-06-30T00:00:00.000Z","profit":67488.3}
+{"id":10,"date":"2022-07-30T00:00:00.000Z","profit":48529.23}
+{"id":10,"date":"2022-08-29T00:00:00.000Z","profit":28680.2}
+{"id":10,"date":"2022-09-28T00:00:00.000Z","profit":59311.16}
+{"id":10,"date":"2022-10-28T00:00:00.000Z","profit":25315.78}
+{"id":10,"date":"2022-11-27T00:00:00.000Z","profit":36116.38}
+{"id":10,"date":"2022-12-27T00:00:00.000Z","profit":-42040.4}

e2e-tests/data/charts-sample/finance_reports.table.yaml

@@ -0,0 +1,15 @@
+name: finance_reports
+columns:
+  - name: id
+    type: int
+    default: null
+    notNull: true
+    references: projects
+  - name: date
+    type: date
+    default: null
+    notNull: true
+  - name: profit
+    type: money
+    default: null
+    notNull: true

e2e-tests/data/charts-sample/projects.jsonl

@@ -0,0 +1,11 @@
+{"__isStreamHeader":true,"pureName":"projects","schemaName":"dbo","objectId":1301579675,"createDate":"2025-06-12T10:30:34.127Z","modifyDate":"2025-06-23T12:15:08.750Z","contentHash":"2025-06-23T12:15:08.750Z","columns":[{"columnName":"id","dataType":"int","notNull":true,"autoIncrement":false,"defaultValue":null,"defaultConstraint":null,"computedExpression":null,"hasAutoValue":false},{"columnName":"name","dataType":"varchar(100)","notNull":true,"autoIncrement":false,"defaultValue":null,"defaultConstraint":null,"computedExpression":null,"hasAutoValue":false},{"columnName":"start_date","dataType":"date","notNull":false,"autoIncrement":false,"defaultValue":null,"defaultConstraint":null,"computedExpression":null,"hasAutoValue":false},{"columnName":"end_date","dataType":"date","notNull":false,"autoIncrement":false,"defaultValue":null,"defaultConstraint":null,"computedExpression":null,"hasAutoValue":false}],"primaryKey":{"constraintName":"PK__projects__3213E83F26A7ED11","schemaName":"dbo","pureName":"projects","constraintType":"primaryKey","columns":[{"columnName":"id"}]},"foreignKeys":[],"indexes":[],"uniques":[],"engine":"mssql@dbgate-plugin-mssql"}
+{"id":1,"name":"Apollo Upgrade","start_date":"2020-04-27T00:00:00.000Z","end_date":"2020-10-19T00:00:00.000Z"}
+{"id":2,"name":"Market Expansion","start_date":"2022-08-04T00:00:00.000Z","end_date":"2023-06-20T00:00:00.000Z"}
+{"id":3,"name":"AI Integration","start_date":"2020-05-11T00:00:00.000Z","end_date":"2021-07-10T00:00:00.000Z"}
+{"id":4,"name":"Cost Reduction","start_date":"2022-01-08T00:00:00.000Z","end_date":"2022-07-12T00:00:00.000Z"}
+{"id":5,"name":"Cloud Migration","start_date":"2021-01-11T00:00:00.000Z","end_date":"2021-05-27T00:00:00.000Z"}
+{"id":6,"name":"Customer Portal","start_date":"2021-07-13T00:00:00.000Z","end_date":"2022-09-22T00:00:00.000Z"}
+{"id":7,"name":"Data Lake","start_date":"2021-02-25T00:00:00.000Z","end_date":"2021-08-21T00:00:00.000Z"}
+{"id":8,"name":"UX Overhaul","start_date":"2021-05-20T00:00:00.000Z","end_date":"2022-09-10T00:00:00.000Z"}
+{"id":9,"name":"Security Hardening","start_date":"2021-05-28T00:00:00.000Z","end_date":"2022-07-28T00:00:00.000Z"}
+{"id":10,"name":"Mobile App Revamp","start_date":"2021-11-17T00:00:00.000Z","end_date":"2022-06-04T00:00:00.000Z"}

e2e-tests/data/charts-sample/projects.table.yaml

@@ -0,0 +1,18 @@
+name: projects
+columns:
+  - name: id
+    type: int
+    default: null
+    notNull: true
+  - name: name
+    type: varchar(100)
+    default: null
+    notNull: true
+  - name: start_date
+    type: date
+    default: null
+  - name: end_date
+    type: date
+    default: null
+primaryKey:
+  - id

e2e-tests/data/files/sql/chart1.sql

@@ -0,0 +1,23 @@
+-- >>>
+-- autoExecute: true
+-- splitterInitialValue: 20%
+-- selected-chart: 1
+-- <<<
+
+SELECT 
+    d.name AS department_name,
+    FORMAT(fr.date, 'yyyy-MM') AS month,
+    SUM(fr.profit) AS total_monthly_profit
+FROM 
+    departments d
+JOIN 
+    employees e ON d.id = e.department_id
+JOIN 
+    employee_project ep ON e.id = ep.employee_id
+JOIN 
+    finance_reports fr ON ep.project_id = fr.id
+GROUP BY 
+    d.name, FORMAT(fr.date, 'yyyy-MM')
+ORDER BY 
+    d.name, month;
+

e2e-tests/env/browse-data/.env

@@ -1,5 +1,4 @@
 CONNECTIONS=mysql,postgres,mongo,redis
-ALLOW_DBGATE_PRIVATE_CLOUD=1
 
 LABEL_mysql=MySql-connection
 SERVER_mysql=localhost

e2e-tests/env/charts/.env

@@ -0,0 +1,8 @@
+CONNECTIONS=mysql
+
+LABEL_mysql=MySql-connection
+SERVER_mysql=localhost
+USER_mysql=root
+PASSWORD_mysql=Pwd2020Db
+PORT_mysql=16004
+ENGINE_mysql=mysql@dbgate-plugin-mysql

e2e-tests/env/cloud/.env

@@ -0,0 +1,2 @@
+ALLOW_DBGATE_PRIVATE_CLOUD=1
+REDIRECT_TO_DBGATE_CLOUD_LOGIN=1

e2e-tests/init/charts.js

@@ -0,0 +1,96 @@
+const path = require('path');
+const os = require('os');
+const fs = require('fs');
+
+const baseDir = path.join(os.homedir(), '.dbgate');
+
+const dbgateApi = require('dbgate-api');
+dbgateApi.initializeApiEnvironment();
+const dbgatePluginMysql = require('dbgate-plugin-mysql');
+dbgateApi.registerPlugins(dbgatePluginMysql);
+
+async function copyFolder(source, target) {
+  if (!fs.existsSync(target)) {
+    fs.mkdirSync(target, { recursive: true });
+  }
+  for (const file of fs.readdirSync(source)) {
+    fs.copyFileSync(path.join(source, file), path.join(target, file));
+  }
+}
+
+async function initMySqlDatabase(dbname, inputFile) {
+  await dbgateApi.executeQuery({
+    connection: {
+      server: process.env.SERVER_mysql,
+      user: process.env.USER_mysql,
+      password: process.env.PASSWORD_mysql,
+      port: process.env.PORT_mysql,
+      engine: 'mysql@dbgate-plugin-mysql',
+    },
+    sql: `drop database if exists ${dbname}`,
+  });
+
+  await dbgateApi.executeQuery({
+    connection: {
+      server: process.env.SERVER_mysql,
+      user: process.env.USER_mysql,
+      password: process.env.PASSWORD_mysql,
+      port: process.env.PORT_mysql,
+      engine: 'mysql@dbgate-plugin-mysql',
+    },
+    sql: `create database ${dbname}`,
+  });
+
+  await dbgateApi.importDatabase({
+    connection: {
+      server: process.env.SERVER_mysql,
+      user: process.env.USER_mysql,
+      password: process.env.PASSWORD_mysql,
+      port: process.env.PORT_mysql,
+      database: dbname,
+      engine: 'mysql@dbgate-plugin-mysql',
+    },
+    inputFile,
+  });
+}
+
+async function run() {
+  const connection = {
+    server: process.env.SERVER_mysql,
+    user: process.env.USER_mysql,
+    password: process.env.PASSWORD_mysql,
+    port: process.env.PORT_mysql,
+    engine: 'mysql@dbgate-plugin-mysql',
+  };
+
+  try {
+    await dbgateApi.executeQuery({
+      connection,
+      sql: 'drop database if exists charts_sample',
+    });
+  } catch (err) {
+    console.error('Failed to drop database', err);
+  }
+
+  await dbgateApi.executeQuery({
+    connection,
+    sql: 'create database charts_sample',
+  });
+
+  await dbgateApi.importDbFromFolder({
+    connection: {
+      ...connection,
+      database: 'charts_sample',
+    },
+    folder: path.resolve(path.join(__dirname, '../data/charts-sample')),
+  });
+
+  await copyFolder(
+    path.resolve(path.join(__dirname, '../data/files/sql')),
+    path.join(baseDir, 'files-e2etests', 'sql')
+  );
+
+  await initMySqlDatabase('MyChinook', path.resolve(path.join(__dirname, '../data/chinook-mysql.sql')));
+}
+
+dbgateApi.runScript(run);

e2e-tests/package.json

@@ -21,6 +21,8 @@
     "cy:run:browse-data": "cypress run --spec cypress/e2e/browse-data.cy.js",
     "cy:run:team": "cypress run --spec cypress/e2e/team.cy.js",
     "cy:run:multi-sql": "cypress run --spec cypress/e2e/multi-sql.cy.js",
+    "cy:run:cloud": "cypress run --spec cypress/e2e/cloud.cy.js",
+    "cy:run:charts": "cypress run --spec cypress/e2e/charts.cy.js",
 
     "start:add-connection": "node clearTestingData && cd .. && node packer/build/bundle.js --listen-api --run-e2e-tests",
     "start:portal": "node clearTestingData && cd .. && env-cmd -f e2e-tests/env/portal/.env node e2e-tests/init/portal.js && env-cmd -f e2e-tests/env/portal/.env node packer/build/bundle.js --listen-api --run-e2e-tests",
@@ -28,6 +30,8 @@
     "start:browse-data": "node clearTestingData && cd .. && env-cmd -f e2e-tests/env/browse-data/.env node e2e-tests/init/browse-data.js && env-cmd -f e2e-tests/env/browse-data/.env node packer/build/bundle.js --listen-api --run-e2e-tests",
     "start:team": "node clearTestingData && cd .. && env-cmd -f e2e-tests/env/team/.env node e2e-tests/init/team.js && env-cmd -f e2e-tests/env/team/.env node packer/build/bundle.js --listen-api --run-e2e-tests",
     "start:multi-sql": "node clearTestingData && cd .. && env-cmd -f e2e-tests/env/multi-sql/.env node e2e-tests/init/multi-sql.js && env-cmd -f e2e-tests/env/multi-sql/.env node packer/build/bundle.js --listen-api --run-e2e-tests",
+    "start:cloud": "node clearTestingData && cd .. && env-cmd -f e2e-tests/env/cloud/.env node packer/build/bundle.js --listen-api --run-e2e-tests",
+    "start:charts": "node clearTestingData && cd .. && env-cmd -f e2e-tests/env/charts/.env node e2e-tests/init/charts.js && env-cmd -f e2e-tests/env/charts/.env node packer/build/bundle.js --listen-api --run-e2e-tests",
 
     "test:add-connection": "start-server-and-test start:add-connection http://localhost:3000 cy:run:add-connection",
     "test:portal": "start-server-and-test start:portal http://localhost:3000 cy:run:portal",
@@ -35,8 +39,10 @@
     "test:browse-data": "start-server-and-test start:browse-data http://localhost:3000 cy:run:browse-data",
     "test:team": "start-server-and-test start:team http://localhost:3000 cy:run:team",
     "test:multi-sql": "start-server-and-test start:multi-sql http://localhost:3000 cy:run:multi-sql",
+    "test:cloud": "start-server-and-test start:cloud http://localhost:3000 cy:run:cloud",
+    "test:charts": "start-server-and-test start:charts http://localhost:3000 cy:run:charts",
 
-    "test": "yarn test:add-connection && yarn test:portal && yarn test:oauth && yarn test:browse-data && yarn test:team && yarn test:multi-sql",
+    "test": "yarn test:add-connection && yarn test:portal && yarn test:oauth && yarn test:browse-data && yarn test:team && yarn test:multi-sql && yarn test:cloud && yarn test:charts",
     "test:ci": "yarn test"
   },
   "dependencies": {}

integration-tests/__tests__/data-replicator.spec.js

@@ -4,7 +4,7 @@ const { testWrapper } = require('../tools');
 const dataReplicator = require('dbgate-api/src/shell/dataReplicator');
 const deployDb = require('dbgate-api/src/shell/deployDb');
 const storageModel = require('dbgate-api/src/storageModel');
-const { runCommandOnDriver, runQueryOnDriver } = require('dbgate-tools');
+const { runCommandOnDriver, runQueryOnDriver, adaptDatabaseInfo } = require('dbgate-tools');
 
 describe('Data replicator', () => {
   test.each(engines.filter(x => !x.skipDataReplicator).map(engine => [engine.label, engine]))(
@@ -162,7 +162,7 @@ describe('Data replicator', () => {
       await deployDb({
         systemConnection: conn,
         driver,
-        loadedDbModel: storageModel,
+        loadedDbModel: adaptDatabaseInfo(storageModel, driver),
         targetSchema: engine.defaultSchemaName,
       });
 
@@ -176,11 +176,11 @@ describe('Data replicator', () => {
         await queryValue(
           `select ~is_disabled as ~val from ~auth_methods where ~amoid='790ca4d2-7f01-4800-955b-d691b890cc50'`
         )
-      ).toBeFalsy();
+      ).toBeTruthy();
 
       const DB1 = {
         auth_methods: [
-          { id: -1, name: 'Anonymous', amoid: '790ca4d2-7f01-4800-955b-d691b890cc50', is_disabled: 1 },
+          { id: -1, name: 'Anonymous', amoid: '790ca4d2-7f01-4800-955b-d691b890cc50', is_disabled: 0 },
           { id: 10, name: 'OAuth', amoid: '4269b660-54b6-11ef-a3aa-a9021250bf4b' },
         ],
         auth_methods_config: [{ id: 20, auth_method_id: 10, key: 'oauthClient', value: 'dbgate' }],
@@ -266,7 +266,7 @@ describe('Data replicator', () => {
         await queryValue(
           `select ~is_disabled as ~val from ~auth_methods where ~amoid='790ca4d2-7f01-4800-955b-d691b890cc50'`
         )
-      ).toBeTruthy();
+      ).toEqual('0');
 
       expect(await queryValue(`select count(*) as ~val from ~auth_methods`)).toEqual('3');
       expect(await queryValue(`select count(*) as ~val from ~auth_methods_config`)).toEqual('1');

package.json

@@ -1,6 +1,6 @@
 {
   "private": true,
-  "version": "6.5.1-beta.1",
+  "version": "6.6.0",
   "name": "dbgate-all",
   "workspaces": [
     "packages/*",
@@ -43,7 +43,7 @@
     "build:plugins:frontend": "workspaces-run --only=\"dbgate-plugin-*\" -- yarn build:frontend",
     "build:plugins:backend": "workspaces-run --only=\"dbgate-plugin-*\" -- yarn build:backend",
     "build:plugins:frontend:watch": "workspaces-run --parallel --only=\"dbgate-plugin-*\" -- yarn build:frontend:watch",
-    "storage-json": "dbmodel model-to-json storage-db packages/api/src/storageModel.js --commonjs",
+    "storage-json": "node packages/dbmodel/bin/dbmodel.js model-to-json storage-db packages/api/src/storageModel.js --commonjs",
     "plugins:copydist": "workspaces-run --only=\"dbgate-plugin-*\" -- yarn copydist",
     "build:app:local": "yarn plugins:copydist && cd app && yarn build:local",
     "start:app:local": "cd app && yarn start:local",

packages/api/.env

@@ -2,6 +2,9 @@ DEVMODE=1
 SHELL_SCRIPTING=1
 ALLOW_DBGATE_PRIVATE_CLOUD=1
 DEVWEB=1 
+# LOCAL_AI_GATEWAY=true
+
+# REDIRECT_TO_DBGATE_CLOUD_LOGIN=1
 # PROD_DBGATE_CLOUD=1
 # PROD_DBGATE_IDENTITY=1
 # LOCAL_DBGATE_CLOUD=1
@@ -13,7 +16,6 @@ DEVWEB=1
 # DISABLE_SHELL=1
 # HIDE_APP_EDITOR=1
 
-
 # DEVWEB=1
 # LOGINS=admin,test
 

packages/api/package.json

@@ -68,6 +68,7 @@
   },
   "scripts": {
     "start": "env-cmd -f .env node src/index.js --listen-api",
+    "start:debug": "env-cmd -f .env node --inspect src/index.js --listen-api",
     "start:portal": "env-cmd -f env/portal/.env node src/index.js --listen-api",
     "start:singledb": "env-cmd -f env/singledb/.env node src/index.js --listen-api",
     "start:auth": "env-cmd -f env/auth/.env node src/index.js --listen-api",

packages/api/src/auth/authProvider.js

@@ -11,7 +11,7 @@ const logger = getLogger('authProvider');
 class AuthProviderBase {
   amoid = 'none';
 
-  async login(login, password, options = undefined) {
+  async login(login, password, options = undefined, req = undefined) {
     return {
       accessToken: jwt.sign(
         {
@@ -23,7 +23,7 @@ class AuthProviderBase {
     };
   }
 
-  oauthToken(params) {
+  oauthToken(params, req) {
     return {};
   }
 

packages/api/src/controllers/auth.js

@@ -13,8 +13,21 @@ const {
 } = require('../auth/authProvider');
 const storage = require('./storage');
 const { decryptPasswordString } = require('../utility/crypting');
-const { createDbGateIdentitySession, startCloudTokenChecking } = require('../utility/cloudIntf');
+const {
+  createDbGateIdentitySession,
+  startCloudTokenChecking,
+  readCloudTokenHolder,
+  readCloudTestTokenHolder,
+} = require('../utility/cloudIntf');
 const socket = require('../utility/socket');
+const { sendToAuditLog } = require('../utility/auditlog');
+const {
+  isLoginLicensed,
+  LOGIN_LIMIT_ERROR,
+  markTokenAsLoggedIn,
+  markUserAsActive,
+  markLoginAsLoggedOut,
+} = require('../utility/loginchecker');
 
 const logger = getLogger('auth');
 
@@ -54,6 +67,11 @@ function authMiddleware(req, res, next) {
 
   // const isAdminPage = req.headers['x-is-admin-page'] == 'true';
 
+  if (process.env.SKIP_ALL_AUTH) {
+    // API is not authorized for basic auth
+    return next();
+  }
+
   if (process.env.BASIC_AUTH) {
     // API is not authorized for basic auth
     return next();
@@ -72,6 +90,8 @@ function authMiddleware(req, res, next) {
   try {
     const decoded = jwt.verify(token, getTokenSecret());
     req.user = decoded;
+    markUserAsActive(decoded.licenseUid, token);
+
     return next();
   } catch (err) {
     if (skipAuth) {
@@ -87,12 +107,12 @@ function authMiddleware(req, res, next) {
 
 module.exports = {
   oauthToken_meta: true,
-  async oauthToken(params) {
+  async oauthToken(params, req) {
     const { amoid } = params;
-    return getAuthProviderById(amoid).oauthToken(params);
+    return getAuthProviderById(amoid).oauthToken(params, req);
   },
   login_meta: true,
-  async login(params) {
+  async login(params, req) {
     const { amoid, login, password, isAdminPage } = params;
 
     if (isAdminPage) {
@@ -102,25 +122,52 @@ module.exports = {
         adminPassword = decryptPasswordString(adminConfig?.adminPassword);
       }
       if (adminPassword && adminPassword == password) {
+        if (!(await isLoginLicensed(req, `superadmin`))) {
+          return { error: LOGIN_LIMIT_ERROR };
+        }
+
+        sendToAuditLog(req, {
+          category: 'auth',
+          component: 'AuthController',
+          action: 'login',
+          event: 'login.admin',
+          severity: 'info',
+          message: 'Administration login successful',
+        });
+
+        const licenseUid = `superadmin`;
+        const accessToken = jwt.sign(
+          {
+            login: 'superadmin',
+            permissions: await storage.loadSuperadminPermissions(),
+            roleId: -3,
+            licenseUid,
+          },
+          getTokenSecret(),
+          {
+            expiresIn: getTokenLifetime(),
+          }
+        );
+        markTokenAsLoggedIn(licenseUid, accessToken);
+
         return {
-          accessToken: jwt.sign(
-            {
-              login: 'superadmin',
-              permissions: await storage.loadSuperadminPermissions(),
-              roleId: -3,
-            },
-            getTokenSecret(),
-            {
-              expiresIn: getTokenLifetime(),
-            }
-          ),
+          accessToken,
         };
       }
 
+      sendToAuditLog(req, {
+        category: 'auth',
+        component: 'AuthController',
+        action: 'loginFail',
+        event: 'login.adminFailed',
+        severity: 'warn',
+        message: 'Administraton login failed',
+      });
+
       return { error: 'Login failed' };
     }
 
-    return getAuthProviderById(amoid).login(login, password);
+    return getAuthProviderById(amoid).login(login, password, undefined, req);
   },
 
   getProviders_meta: true,
@@ -138,13 +185,39 @@ module.exports = {
   },
 
   createCloudLoginSession_meta: true,
-  async createCloudLoginSession({ client }) {
-    const res = await createDbGateIdentitySession(client);
+  async createCloudLoginSession({ client, redirectUri }) {
+    const res = await createDbGateIdentitySession(client, redirectUri);
     startCloudTokenChecking(res.sid, tokenHolder => {
       socket.emit('got-cloud-token', tokenHolder);
+      socket.emitChanged('cloud-content-changed');
+      socket.emit('cloud-content-updated');
     });
     return res;
   },
 
+  cloudLoginRedirected_meta: true,
+  async cloudLoginRedirected({ sid }) {
+    const tokenHolder = await readCloudTokenHolder(sid);
+    return tokenHolder;
+  },
+
+  cloudTestLogin_meta: true,
+  async cloudTestLogin({ email }) {
+    const tokenHolder = await readCloudTestTokenHolder(email);
+    return tokenHolder;
+  },
+
+  logoutAdmin_meta: true,
+  async logoutAdmin() {
+    await markLoginAsLoggedOut('superadmin');
+    return true;
+  },
+
+  logoutUser_meta: true,
+  async logoutUser({}, req) {
+    await markLoginAsLoggedOut(req?.user?.licenseUid);
+    return true;
+  },
+
   authMiddleware,
 };

packages/api/src/controllers/cloud.js

@@ -16,6 +16,7 @@ const { getConnectionLabel, getLogger, extractErrorLogData } = require('dbgate-t
 const logger = getLogger('cloud');
 const _ = require('lodash');
 const fs = require('fs-extra');
+const { getAiGatewayServer } = require('../utility/authProxy');
 
 module.exports = {
   publicFiles_meta: true,
@@ -258,4 +259,35 @@ module.exports = {
     await fs.writeFile(filePath, content);
     return true;
   },
+
+  folderUsers_meta: true,
+  async folderUsers({ folid }) {
+    const resp = await callCloudApiGet(`content-folders/users/${folid}`);
+    return resp;
+  },
+
+  setFolderUserRole_meta: true,
+  async setFolderUserRole({ folid, email, role }) {
+    const resp = await callCloudApiPost(`content-folders/set-user-role/${folid}`, { email, role });
+    return resp;
+  },
+
+  removeFolderUser_meta: true,
+  async removeFolderUser({ folid, email }) {
+    const resp = await callCloudApiPost(`content-folders/remove-user/${folid}`, { email });
+    return resp;
+  },
+
+  getAiGateway_meta: true,
+  async getAiGateway() {
+    return getAiGatewayServer();
+  },
+
+  // chatStream_meta: {
+  //   raw: true,
+  //   method: 'post',
+  // },
+  // chatStream(req, res) {
+  //   callChatStream(req.body, res);
+  // },
 };

packages/api/src/controllers/config.js

@@ -16,7 +16,7 @@ const connections = require('../controllers/connections');
 const { getAuthProviderFromReq } = require('../auth/authProvider');
 const { checkLicense, checkLicenseKey } = require('../utility/checkLicense');
 const storage = require('./storage');
-const { getAuthProxyUrl } = require('../utility/authProxy');
+const { getAuthProxyUrl, tryToGetRefreshedLicense } = require('../utility/authProxy');
 const { getPublicHardwareFingerprint } = require('../utility/hardwareFingerprint');
 const { extractErrorMessage } = require('dbgate-tools');
 const {
@@ -29,6 +29,7 @@ const {
 } = require('../utility/crypting');
 
 const lock = new AsyncLock();
+let cachedSettingsValue = null;
 
 module.exports = {
   // settingsValue: {},
@@ -108,6 +109,7 @@ module.exports = {
       ),
       isAdminPasswordMissing,
       isInvalidToken: req?.isInvalidToken,
+      skipAllAuth: !!process.env.SKIP_ALL_AUTH,
       adminPasswordState: adminConfig?.adminPasswordState,
       storageDatabase: process.env.STORAGE_DATABASE,
       logsFilePath: getLogsFilePath(),
@@ -118,6 +120,7 @@ module.exports = {
       supportCloudAutoUpgrade: !!process.env.CLOUD_UPGRADE_FILE,
       allowPrivateCloud: platformInfo.isElectron || !!process.env.ALLOW_DBGATE_PRIVATE_CLOUD,
       ...currentVersion,
+      redirectToDbGateCloudLogin: !!process.env.REDIRECT_TO_DBGATE_CLOUD_LOGIN,
     };
 
     return configResult;
@@ -144,6 +147,13 @@ module.exports = {
     return res;
   },
 
+  async getCachedSettings() {
+    if (!cachedSettingsValue) {
+      cachedSettingsValue = await this.loadSettings();
+    }
+    return cachedSettingsValue;
+  },
+
   deleteSettings_meta: true,
   async deleteSettings() {
     await fs.unlink(path.join(datadir(), processArgs.runE2eTests ? 'settings-e2etests.json' : 'settings.json'));
@@ -182,6 +192,7 @@ module.exports = {
         return {
           ...this.fillMissingSettings(JSON.parse(settingsText)),
           'other.licenseKey': platformInfo.isElectron ? await this.loadLicenseKey() : undefined,
+          // 'other.licenseKey': await this.loadLicenseKey(),
         };
       }
     } catch (err) {
@@ -199,21 +210,34 @@ module.exports = {
   },
 
   saveLicenseKey_meta: true,
-  async saveLicenseKey({ licenseKey }) {
-    const decoded = jwt.decode(licenseKey?.trim());
-    if (!decoded) {
-      return {
-        status: 'error',
-        errorMessage: 'Invalid license key',
-      };
-    }
+  async saveLicenseKey({ licenseKey, forceSave = false, tryToRenew = false }) {
+    if (!forceSave) {
+      const decoded = jwt.decode(licenseKey?.trim());
+      if (!decoded) {
+        return {
+          status: 'error',
+          errorMessage: 'Invalid license key',
+        };
+      }
 
-    const { exp } = decoded;
-    if (exp * 1000 < Date.now()) {
-      return {
-        status: 'error',
-        errorMessage: 'License key is expired',
-      };
+      const { exp } = decoded;
+      if (exp * 1000 < Date.now()) {
+        let renewed = false;
+        if (tryToRenew) {
+          const newLicenseKey = await tryToGetRefreshedLicense(licenseKey);
+          if (newLicenseKey.status == 'ok') {
+            licenseKey = newLicenseKey.token;
+            renewed = true;
+          }
+        }
+
+        if (!renewed) {
+          return {
+            status: 'error',
+            errorMessage: 'License key is expired',
+          };
+        }
+      }
     }
 
     try {
@@ -257,6 +281,7 @@ module.exports = {
   updateSettings_meta: true,
   async updateSettings(values, req) {
     if (!hasPermission(`settings/change`, req)) return false;
+    cachedSettingsValue = null;
 
     const res = await lock.acquire('settings', async () => {
       const currentValue = await this.loadSettings();
@@ -265,7 +290,11 @@ module.exports = {
         if (process.env.STORAGE_DATABASE) {
           updated = {
             ...currentValue,
-            ...values,
+            ..._.mapValues(values, v => {
+              if (v === true) return 'true';
+              if (v === false) return 'false';
+              return v;
+            }),
           };
           await storage.writeConfig({
             group: 'settings',
@@ -283,7 +312,7 @@ module.exports = {
           // this.settingsValue = updated;
 
           if (currentValue['other.licenseKey'] != values['other.licenseKey']) {
-            await this.saveLicenseKey({ licenseKey: values['other.licenseKey'] });
+            await this.saveLicenseKey({ licenseKey: values['other.licenseKey'], forceSave: true });
             socket.emitChanged(`config-changed`);
           }
         }
@@ -303,7 +332,7 @@ module.exports = {
       const resp = await axios.default.get('https://raw.githubusercontent.com/dbgate/dbgate/master/CHANGELOG.md');
       return resp.data;
     } catch (err) {
-      return ''
+      return '';
     }
   },
 
@@ -313,6 +342,16 @@ module.exports = {
     return resp;
   },
 
+  getNewLicense_meta: true,
+  async getNewLicense({ oldLicenseKey }) {
+    const newLicenseKey = await tryToGetRefreshedLicense(oldLicenseKey);
+    const res = await checkLicenseKey(newLicenseKey.token);
+    if (res.status == 'ok') {
+      res.licenseKey = newLicenseKey.token;
+    }
+    return res;
+  },
+
   recryptDatabaseForExport(db) {
     const encryptionKey = generateTransportEncryptionKey();
     const transportEncryptor = createTransportEncryptor(encryptionKey);

packages/api/src/controllers/connections.js

@@ -536,14 +536,14 @@ module.exports = {
   },
 
   dbloginAuthToken_meta: true,
-  async dbloginAuthToken({ amoid, code, conid, redirectUri, sid }) {
+  async dbloginAuthToken({ amoid, code, conid, redirectUri, sid }, req) {
     try {
       const connection = await this.getCore({ conid });
       const driver = requireEngineDriver(connection);
       const accessToken = await driver.getAuthTokenFromCode(connection, { code, redirectUri, sid });
       const volatile = await this.saveVolatile({ conid, accessToken });
       const authProvider = getAuthProviderById(amoid);
-      const resp = await authProvider.login(null, null, { conid: volatile._id });
+      const resp = await authProvider.login(null, null, { conid: volatile._id }, req);
       return resp;
     } catch (err) {
       logger.error(extractErrorLogData(err), 'Error getting DB token');
@@ -552,18 +552,18 @@ module.exports = {
   },
 
   dbloginAuth_meta: true,
-  async dbloginAuth({ amoid, conid, user, password }) {
+  async dbloginAuth({ amoid, conid, user, password }, req) {
     if (user || password) {
       const saveResp = await this.saveVolatile({ conid, user, password, test: true });
       if (saveResp.msgtype == 'connected') {
-        const loginResp = await getAuthProviderById(amoid).login(user, password, { conid: saveResp._id });
+        const loginResp = await getAuthProviderById(amoid).login(user, password, { conid: saveResp._id }, req);
         return loginResp;
       }
       return saveResp;
     }
 
     // user and password is stored in connection, volatile connection is not needed
-    const loginResp = await getAuthProviderById(amoid).login(null, null, { conid });
+    const loginResp = await getAuthProviderById(amoid).login(null, null, { conid }, req);
     return loginResp;
   },
 

packages/api/src/controllers/databaseConnections.js

@@ -41,6 +41,7 @@ const { decryptConnection } = require('../utility/crypting');
 const { getSshTunnel } = require('../utility/sshTunnel');
 const sessions = require('./sessions');
 const jsldata = require('./jsldata');
+const { sendToAuditLog } = require('../utility/auditlog');
 
 const logger = getLogger('databaseConnections');
 
@@ -83,8 +84,11 @@ module.exports = {
     }
   },
   handle_response(conid, database, { msgid, ...response }) {
-    const [resolve, reject] = this.requests[msgid];
+    const [resolve, reject, additionalData] = this.requests[msgid];
     resolve(response);
+    if (additionalData?.auditLogger) {
+      additionalData?.auditLogger(response);
+    }
     delete this.requests[msgid];
   },
   handle_status(conid, database, { status }) {
@@ -215,10 +219,10 @@ module.exports = {
   },
 
   /** @param {import('dbgate-types').OpenedDatabaseConnection} conn */
-  sendRequest(conn, message) {
+  sendRequest(conn, message, additionalData = {}) {
     const msgid = crypto.randomUUID();
     const promise = new Promise((resolve, reject) => {
-      this.requests[msgid] = [resolve, reject];
+      this.requests[msgid] = [resolve, reject, additionalData];
       try {
         conn.subprocess.send({ msgid, ...message });
       } catch (err) {
@@ -242,18 +246,57 @@ module.exports = {
   },
 
   sqlSelect_meta: true,
-  async sqlSelect({ conid, database, select }, req) {
+  async sqlSelect({ conid, database, select, auditLogSessionGroup }, req) {
     testConnectionPermission(conid, req);
     const opened = await this.ensureOpened(conid, database);
-    const res = await this.sendRequest(opened, { msgtype: 'sqlSelect', select });
+    const res = await this.sendRequest(
+      opened,
+      { msgtype: 'sqlSelect', select },
+      {
+        auditLogger:
+          auditLogSessionGroup && select?.from?.name?.pureName
+            ? response => {
+                sendToAuditLog(req, {
+                  category: 'dbop',
+                  component: 'DatabaseConnectionsController',
+                  event: 'sql.select',
+                  action: 'select',
+                  severity: 'info',
+                  conid,
+                  database,
+                  schemaName: select?.from?.name?.schemaName,
+                  pureName: select?.from?.name?.pureName,
+                  sumint1: response?.rows?.length,
+                  sessionParam: `${conid}::${database}::${select?.from?.name?.schemaName || '0'}::${
+                    select?.from?.name?.pureName
+                  }`,
+                  sessionGroup: auditLogSessionGroup,
+                  message: `Loaded table data from ${select?.from?.name?.pureName}`,
+                });
+              }
+            : null,
+      }
+    );
     return res;
   },
 
   runScript_meta: true,
-  async runScript({ conid, database, sql, useTransaction }, req) {
+  async runScript({ conid, database, sql, useTransaction, logMessage }, req) {
     testConnectionPermission(conid, req);
     logger.info({ conid, database, sql }, 'Processing script');
     const opened = await this.ensureOpened(conid, database);
+    sendToAuditLog(req, {
+      category: 'dbop',
+      component: 'DatabaseConnectionsController',
+      event: 'sql.runscript',
+      action: 'runscript',
+      severity: 'info',
+      conid,
+      database,
+      detail: sql,
+      message: logMessage || `Running SQL script`,
+    });
+
     const res = await this.sendRequest(opened, { msgtype: 'runScript', sql, useTransaction });
     return res;
   },
@@ -262,16 +305,53 @@ module.exports = {
   async runOperation({ conid, database, operation, useTransaction }, req) {
     testConnectionPermission(conid, req);
     logger.info({ conid, database, operation }, 'Processing operation');
+
+    sendToAuditLog(req, {
+      category: 'dbop',
+      component: 'DatabaseConnectionsController',
+      event: 'sql.runoperation',
+      action: operation.type,
+      severity: 'info',
+      conid,
+      database,
+      detail: operation,
+      message: `Running DB operation: ${operation.type}`,
+    });
+
     const opened = await this.ensureOpened(conid, database);
     const res = await this.sendRequest(opened, { msgtype: 'runOperation', operation, useTransaction });
     return res;
   },
 
   collectionData_meta: true,
-  async collectionData({ conid, database, options }, req) {
+  async collectionData({ conid, database, options, auditLogSessionGroup }, req) {
     testConnectionPermission(conid, req);
     const opened = await this.ensureOpened(conid, database);
-    const res = await this.sendRequest(opened, { msgtype: 'collectionData', options });
+    const res = await this.sendRequest(
+      opened,
+      { msgtype: 'collectionData', options },
+      {
+        auditLogger:
+          auditLogSessionGroup && options?.pureName
+            ? response => {
+                sendToAuditLog(req, {
+                  category: 'dbop',
+                  component: 'DatabaseConnectionsController',
+                  event: 'nosql.collectionData',
+                  action: 'select',
+                  severity: 'info',
+                  conid,
+                  database,
+                  pureName: options?.pureName,
+                  sumint1: response?.result?.rows?.length,
+                  sessionParam: `${conid}::${database}::${options?.pureName}`,
+                  sessionGroup: auditLogSessionGroup,
+                  message: `Loaded collection data ${options?.pureName}`,
+                });
+              }
+            : null,
+      }
+    );
     return res.result || null;
   },
 
@@ -492,6 +572,20 @@ module.exports = {
     }
 
     const opened = await this.ensureOpened(conid, database);
+
+    sendToAuditLog(req, {
+      category: 'dbop',
+      component: 'DatabaseConnectionsController',
+      action: 'structure',
+      event: 'dbStructure.get',
+      severity: 'info',
+      conid,
+      database,
+      sessionParam: `${conid}::${database}`,
+      sessionGroup: 'getStructure',
+      message: `Loaded database structure for ${database}`,
+    });
+
     return opened.structure;
     // const existing = this.opened.find((x) => x.conid == conid && x.database == database);
     // if (existing) return existing.status;

packages/api/src/controllers/files.js

@@ -203,10 +203,10 @@ module.exports = {
   },
 
   exportChart_meta: true,
-  async exportChart({ filePath, title, config, image }) {
+  async exportChart({ filePath, title, config, image, plugins }) {
     const fileName = path.parse(filePath).base;
     const imageFile = fileName.replace('.html', '-preview.png');
-    const html = getChartExport(title, config, imageFile);
+    const html = getChartExport(title, config, imageFile, plugins);
     await fs.writeFile(filePath, html);
     if (image) {
       const index = image.indexOf('base64,');

packages/api/src/controllers/jsldata.js

@@ -313,19 +313,9 @@ module.exports = {
       return true;
     });
     processor.finalize();
-    return processor.charts;
-  },
-
-  detectChartColumns_meta: true,
-  async detectChartColumns({ jslid }) {
-    const datastore = new JsonLinesDatastore(getJslFileName(jslid));
-    const processor = new ChartProcessor();
-    processor.autoDetectCharts = false;
-    await datastore.enumRows(row => {
-      processor.addRow(row);
-      return true;
-    });
-    processor.finalize();
-    return processor.availableColumns;
+    return {
+      charts: processor.charts,
+      columns: processor.availableColumns,
+    };
   },
 };

packages/api/src/controllers/runners.js

@@ -20,6 +20,7 @@ const { handleProcessCommunication } = require('../utility/processComm');
 const processArgs = require('../utility/processArgs');
 const platformInfo = require('../utility/platformInfo');
 const { checkSecureDirectories, checkSecureDirectoriesInScript } = require('../utility/security');
+const { sendToAuditLog, logJsonRunnerScript } = require('../utility/auditlog');
 const logger = getLogger('runners');
 
 function extractPlugins(script) {
@@ -270,7 +271,7 @@ module.exports = {
   },
 
   start_meta: true,
-  async start({ script }) {
+  async start({ script }, req) {
     const runid = crypto.randomUUID();
 
     if (script.type == 'json') {
@@ -280,14 +281,36 @@ module.exports = {
         }
       }
 
+      logJsonRunnerScript(req, script);
+
       const js = await jsonScriptToJavascript(script);
       return this.startCore(runid, scriptTemplate(js, false));
     }
 
     if (!platformInfo.allowShellScripting) {
+      sendToAuditLog(req, {
+        category: 'shell',
+        component: 'RunnersController',
+        event: 'script.runFailed',
+        action: 'script',
+        severity: 'warn',
+        detail: script,
+        message: 'Scripts are not allowed',
+      });
+
       return { errorMessage: 'Shell scripting is not allowed' };
     }
 
+    sendToAuditLog(req, {
+      category: 'shell',
+      component: 'RunnersController',
+      event: 'script.run.shell',
+      action: 'script',
+      severity: 'info',
+      detail: script,
+      message: 'Running JS script',
+    });
+
     return this.startCore(runid, scriptTemplate(script, false));
   },
 

packages/api/src/controllers/serverConnections.js

@@ -12,6 +12,7 @@ const { testConnectionPermission } = require('../utility/hasPermission');
 const { MissingCredentialsError } = require('../utility/exceptions');
 const pipeForkLogs = require('../utility/pipeForkLogs');
 const { getLogger, extractErrorLogData } = require('dbgate-tools');
+const { sendToAuditLog } = require('../utility/auditlog');
 
 const logger = getLogger('serverConnection');
 
@@ -145,6 +146,17 @@ module.exports = {
     if (conid == '__model') return [];
     testConnectionPermission(conid, req);
     const opened = await this.ensureOpened(conid);
+    sendToAuditLog(req, {
+      category: 'serverop',
+      component: 'ServerConnectionsController',
+      action: 'listDatabases',
+      event: 'databases.list',
+      severity: 'info',
+      conid,
+      sessionParam: `${conid}`,
+      sessionGroup: 'listDatabases',
+      message: `Loaded databases for connection`,
+    });
     return opened?.databases ?? [];
   },
 

packages/api/src/controllers/sessions.js

@@ -11,6 +11,7 @@ const { appdir } = require('../utility/directories');
 const { getLogger, extractErrorLogData } = require('dbgate-tools');
 const pipeForkLogs = require('../utility/pipeForkLogs');
 const config = require('./config');
+const { sendToAuditLog } = require('../utility/auditlog');
 
 const logger = getLogger('sessions');
 
@@ -146,15 +147,34 @@ module.exports = {
   },
 
   executeQuery_meta: true,
-  async executeQuery({ sesid, sql, autoCommit, limitRows, frontMatter }) {
+  async executeQuery({ sesid, sql, autoCommit, autoDetectCharts, limitRows, frontMatter }, req) {
     const session = this.opened.find(x => x.sesid == sesid);
     if (!session) {
       throw new Error('Invalid session');
     }
 
+    sendToAuditLog(req, {
+      category: 'dbop',
+      component: 'SessionController',
+      action: 'executeQuery',
+      event: 'query.execute',
+      severity: 'info',
+      detail: sql,
+      conid: session.conid,
+      database: session.database,
+      message: 'Executing query',
+    });
+
     logger.info({ sesid, sql }, 'Processing query');
     this.dispatchMessage(sesid, 'Query execution started');
-    session.subprocess.send({ msgtype: 'executeQuery', sql, autoCommit, limitRows, frontMatter });
+    session.subprocess.send({
+      msgtype: 'executeQuery',
+      sql,
+      autoCommit,
+      autoDetectCharts: autoDetectCharts || !!frontMatter?.['selected-chart'],
+      limitRows,
+      frontMatter,
+    });
 
     return { state: 'ok' };
   },

packages/api/src/controllers/storage.js

@@ -31,6 +31,11 @@ module.exports = {
     return {};
   },
 
+  sendAuditLog_meta: true,
+  async sendAuditLog({}) {
+    return null;
+  },
+
   startRefreshLicense() {},
 
   async getUsedEngines() {

packages/api/src/proc/sessionProcess.js

@@ -117,7 +117,7 @@ async function handleExecuteControlCommand({ command }) {
   }
 }
 
-async function handleExecuteQuery({ sql, autoCommit, limitRows, frontMatter }) {
+async function handleExecuteQuery({ sql, autoCommit, autoDetectCharts, limitRows, frontMatter }) {
   lastActivity = new Date().getTime();
 
   await waitConnected();
@@ -146,7 +146,16 @@ async function handleExecuteQuery({ sql, autoCommit, limitRows, frontMatter }) {
       ...driver.getQuerySplitterOptions('stream'),
       returnRichInfo: true,
     })) {
-      await handleQueryStream(dbhan, driver, queryStreamInfoHolder, sqlItem, undefined, limitRows, frontMatter);
+      await handleQueryStream(
+        dbhan,
+        driver,
+        queryStreamInfoHolder,
+        sqlItem,
+        undefined,
+        limitRows,
+        frontMatter,
+        autoDetectCharts
+      );
       // const handler = new StreamHandler(resultIndex);
       // const stream = await driver.stream(systemConnection, sqlItem, handler);
       // handler.stream = stream;

packages/api/src/shell/deployDb.js

@@ -14,7 +14,7 @@ const crypto = require('crypto');
  * @param {object} options.driver - driver object. If not provided, it will be loaded from connection
  * @param {object} options.analysedStructure - analysed structure of the database. If not provided, it will be loaded
  * @param {string} options.modelFolder - folder with model files (YAML files for tables, SQL files for views, procedures, ...)
- * @param {import('dbgate-tools').DatabaseModelFile[]} options.loadedDbModel - loaded database model - collection of yaml and SQL files loaded into array
+ * @param {import('dbgate-tools').DatabaseModelFile[] | import('dbgate-types').DatabaseInfo} options.loadedDbModel - loaded database model - collection of yaml and SQL files loaded into array
  * @param {function[]} options.modelTransforms - array of functions for transforming model
  * @param {object} options.dbdiffOptionsExtra - extra options for dbdiff
  * @param {string} options.ignoreNameRegex - regex for ignoring objects by name

packages/api/src/shell/generateDeploySql.js

@@ -23,7 +23,7 @@ const { connectUtility } = require('../utility/connectUtility');
  * @param {object} options.driver - driver object. If not provided, it will be loaded from connection
  * @param {object} options.analysedStructure - analysed structure of the database. If not provided, it will be loaded
  * @param {string} options.modelFolder - folder with model files (YAML files for tables, SQL files for views, procedures, ...)
- * @param {import('dbgate-tools').DatabaseModelFile[]} options.loadedDbModel - loaded database model - collection of yaml and SQL files loaded into array
+ * @param {import('dbgate-tools').DatabaseModelFile[] | import('dbgate-types').DatabaseInfo} options.loadedDbModel - loaded database model - collection of yaml and SQL files loaded into array
  * @param {function[]} options.modelTransforms - array of functions for transforming model
  * @param {object} options.dbdiffOptionsExtra - extra options for dbdiff
  * @param {string} options.ignoreNameRegex - regex for ignoring objects by name

packages/api/src/shell/importDbFromFolder.js

@@ -3,7 +3,7 @@ const fs = require('fs-extra');
 const executeQuery = require('./executeQuery');
 const { connectUtility } = require('../utility/connectUtility');
 const requireEngineDriver = require('../utility/requireEngineDriver');
-const { getAlterDatabaseScript, DatabaseAnalyser, runCommandOnDriver } = require('dbgate-tools');
+const { getAlterDatabaseScript, DatabaseAnalyser, runCommandOnDriver, adaptDatabaseInfo } = require('dbgate-tools');
 const importDbModel = require('../utility/importDbModel');
 const jsonLinesReader = require('./jsonLinesReader');
 const tableWriter = require('./tableWriter');
@@ -26,10 +26,7 @@ async function importDbFromFolder({ connection, systemConnection, driver, folder
     if (driver?.databaseEngineTypes?.includes('sql')) {
       const model = await importDbModel(folder);
 
-      let modelAdapted = {
-        ...model,
-        tables: model.tables.map(table => driver.adaptTableInfo(table)),
-      };
+      let modelAdapted = adaptDatabaseInfo(model, driver);
       for (const transform of modelTransforms || []) {
         modelAdapted = transform(modelAdapted);
       }

packages/api/src/storageModel.js

@@ -1,5 +1,192 @@
 module.exports = {
   "tables": [
+    {
+      "pureName": "audit_log",
+      "columns": [
+        {
+          "pureName": "audit_log",
+          "columnName": "id",
+          "dataType": "int",
+          "autoIncrement": true,
+          "notNull": true
+        },
+        {
+          "pureName": "audit_log",
+          "columnName": "created",
+          "dataType": "bigint",
+          "notNull": true
+        },
+        {
+          "pureName": "audit_log",
+          "columnName": "modified",
+          "dataType": "bigint",
+          "notNull": false
+        },
+        {
+          "pureName": "audit_log",
+          "columnName": "user_id",
+          "dataType": "int",
+          "notNull": false
+        },
+        {
+          "pureName": "audit_log",
+          "columnName": "user_login",
+          "dataType": "varchar(250)",
+          "notNull": false
+        },
+        {
+          "pureName": "audit_log",
+          "columnName": "category",
+          "dataType": "varchar(50)",
+          "notNull": false
+        },
+        {
+          "pureName": "audit_log",
+          "columnName": "component",
+          "dataType": "varchar(50)",
+          "notNull": false
+        },
+        {
+          "pureName": "audit_log",
+          "columnName": "action",
+          "dataType": "varchar(50)",
+          "notNull": false
+        },
+        {
+          "pureName": "audit_log",
+          "columnName": "severity",
+          "dataType": "varchar(50)",
+          "notNull": false
+        },
+        {
+          "pureName": "audit_log",
+          "columnName": "event",
+          "dataType": "varchar(100)",
+          "notNull": false
+        },
+        {
+          "pureName": "audit_log",
+          "columnName": "message",
+          "dataType": "varchar(250)",
+          "notNull": false
+        },
+        {
+          "pureName": "audit_log",
+          "columnName": "detail",
+          "dataType": "varchar(1000)",
+          "notNull": false
+        },
+        {
+          "pureName": "audit_log",
+          "columnName": "detail_full_length",
+          "dataType": "int",
+          "notNull": false
+        },
+        {
+          "pureName": "audit_log",
+          "columnName": "session_id",
+          "dataType": "varchar(200)",
+          "notNull": false
+        },
+        {
+          "pureName": "audit_log",
+          "columnName": "session_group",
+          "dataType": "varchar(50)",
+          "notNull": false
+        },
+        {
+          "pureName": "audit_log",
+          "columnName": "session_param",
+          "dataType": "varchar(200)",
+          "notNull": false
+        },
+        {
+          "pureName": "audit_log",
+          "columnName": "conid",
+          "dataType": "varchar(100)",
+          "notNull": false
+        },
+        {
+          "pureName": "audit_log",
+          "columnName": "connection_data",
+          "dataType": "varchar(1000)",
+          "notNull": false
+        },
+        {
+          "pureName": "audit_log",
+          "columnName": "database",
+          "dataType": "varchar(200)",
+          "notNull": false
+        },
+        {
+          "pureName": "audit_log",
+          "columnName": "schema_name",
+          "dataType": "varchar(100)",
+          "notNull": false
+        },
+        {
+          "pureName": "audit_log",
+          "columnName": "pure_name",
+          "dataType": "varchar(100)",
+          "notNull": false
+        },
+        {
+          "pureName": "audit_log",
+          "columnName": "sumint_1",
+          "dataType": "int",
+          "notNull": false
+        },
+        {
+          "pureName": "audit_log",
+          "columnName": "sumint_2",
+          "dataType": "int",
+          "notNull": false
+        }
+      ],
+      "foreignKeys": [
+        {
+          "constraintType": "foreignKey",
+          "constraintName": "FK_audit_log_user_id",
+          "pureName": "audit_log",
+          "refTableName": "users",
+          "deleteAction": "SET NULL",
+          "columns": [
+            {
+              "columnName": "user_id",
+              "refColumnName": "id"
+            }
+          ]
+        }
+      ],
+      "indexes": [
+        {
+          "constraintName": "idx_audit_log_session",
+          "pureName": "audit_log",
+          "constraintType": "index",
+          "columns": [
+            {
+              "columnName": "session_group"
+            },
+            {
+              "columnName": "session_id"
+            },
+            {
+              "columnName": "session_param"
+            }
+          ]
+        }
+      ],
+      "primaryKey": {
+        "pureName": "audit_log",
+        "constraintType": "primaryKey",
+        "constraintName": "PK_audit_log",
+        "columns": [
+          {
+            "columnName": "id"
+          }
+        ]
+      }
+    },
     {
       "pureName": "auth_methods",
       "columns": [
@@ -50,6 +237,7 @@ module.exports = {
       "primaryKey": {
         "pureName": "auth_methods",
         "constraintType": "primaryKey",
+        "constraintName": "PK_auth_methods",
         "columns": [
           {
             "columnName": "id"
@@ -61,7 +249,8 @@ module.exports = {
           "id": -1,
           "amoid": "790ca4d2-7f01-4800-955b-d691b890cc50",
           "name": "Anonymous",
-          "type": "none"
+          "type": "none",
+          "is_disabled": 1
         },
         {
           "id": -2,
@@ -69,6 +258,9 @@ module.exports = {
           "name": "Local",
           "type": "local"
         }
+      ],
+      "preloadedRowsInsertOnly": [
+        "is_disabled"
       ]
     },
     {
@@ -103,6 +295,7 @@ module.exports = {
       "foreignKeys": [
         {
           "constraintType": "foreignKey",
+          "constraintName": "FK_auth_methods_config_auth_method_id",
           "pureName": "auth_methods_config",
           "refTableName": "auth_methods",
           "deleteAction": "CASCADE",
@@ -114,9 +307,25 @@ module.exports = {
           ]
         }
       ],
+      "uniques": [
+        {
+          "constraintName": "UQ_auth_methods_config_auth_method_id_key",
+          "pureName": "auth_methods_config",
+          "constraintType": "unique",
+          "columns": [
+            {
+              "columnName": "auth_method_id"
+            },
+            {
+              "columnName": "key"
+            }
+          ]
+        }
+      ],
       "primaryKey": {
         "pureName": "auth_methods_config",
         "constraintType": "primaryKey",
+        "constraintName": "PK_auth_methods_config",
         "columns": [
           {
             "columnName": "id"
@@ -154,9 +363,25 @@ module.exports = {
         }
       ],
       "foreignKeys": [],
+      "uniques": [
+        {
+          "constraintName": "UQ_config_group_key",
+          "pureName": "config",
+          "constraintType": "unique",
+          "columns": [
+            {
+              "columnName": "group"
+            },
+            {
+              "columnName": "key"
+            }
+          ]
+        }
+      ],
       "primaryKey": {
         "pureName": "config",
         "constraintType": "primaryKey",
+        "constraintName": "PK_config",
         "columns": [
           {
             "columnName": "id"
@@ -294,6 +519,12 @@ module.exports = {
           "dataType": "int",
           "notNull": false
         },
+        {
+          "pureName": "connections",
+          "columnName": "useSeparateSchemas",
+          "dataType": "int",
+          "notNull": false
+        },
         {
           "pureName": "connections",
           "columnName": "defaultDatabase",
@@ -443,12 +674,19 @@ module.exports = {
           "columnName": "awsRegion",
           "dataType": "varchar(250)",
           "notNull": false
+        },
+        {
+          "pureName": "connections",
+          "columnName": "connectionDefinition",
+          "dataType": "text",
+          "notNull": false
         }
       ],
       "foreignKeys": [],
       "primaryKey": {
         "pureName": "connections",
         "constraintType": "primaryKey",
+        "constraintName": "PK_connections",
         "columns": [
           {
             "columnName": "id"
@@ -477,6 +715,7 @@ module.exports = {
       "primaryKey": {
         "pureName": "roles",
         "constraintType": "primaryKey",
+        "constraintName": "PK_roles",
         "columns": [
           {
             "columnName": "id"
@@ -524,6 +763,7 @@ module.exports = {
       "foreignKeys": [
         {
           "constraintType": "foreignKey",
+          "constraintName": "FK_role_connections_role_id",
           "pureName": "role_connections",
           "refTableName": "roles",
           "deleteAction": "CASCADE",
@@ -536,6 +776,7 @@ module.exports = {
         },
         {
           "constraintType": "foreignKey",
+          "constraintName": "FK_role_connections_connection_id",
           "pureName": "role_connections",
           "refTableName": "connections",
           "deleteAction": "CASCADE",
@@ -550,6 +791,7 @@ module.exports = {
       "primaryKey": {
         "pureName": "role_connections",
         "constraintType": "primaryKey",
+        "constraintName": "PK_role_connections",
         "columns": [
           {
             "columnName": "id"
@@ -583,6 +825,7 @@ module.exports = {
       "foreignKeys": [
         {
           "constraintType": "foreignKey",
+          "constraintName": "FK_role_permissions_role_id",
           "pureName": "role_permissions",
           "refTableName": "roles",
           "deleteAction": "CASCADE",
@@ -597,6 +840,7 @@ module.exports = {
       "primaryKey": {
         "pureName": "role_permissions",
         "constraintType": "primaryKey",
+        "constraintName": "PK_role_permissions",
         "columns": [
           {
             "columnName": "id"
@@ -637,6 +881,7 @@ module.exports = {
       "primaryKey": {
         "pureName": "users",
         "constraintType": "primaryKey",
+        "constraintName": "PK_users",
         "columns": [
           {
             "columnName": "id"
@@ -670,6 +915,7 @@ module.exports = {
       "foreignKeys": [
         {
           "constraintType": "foreignKey",
+          "constraintName": "FK_user_connections_user_id",
           "pureName": "user_connections",
           "refTableName": "users",
           "deleteAction": "CASCADE",
@@ -682,6 +928,7 @@ module.exports = {
         },
         {
           "constraintType": "foreignKey",
+          "constraintName": "FK_user_connections_connection_id",
           "pureName": "user_connections",
           "refTableName": "connections",
           "deleteAction": "CASCADE",
@@ -696,6 +943,7 @@ module.exports = {
       "primaryKey": {
         "pureName": "user_connections",
         "constraintType": "primaryKey",
+        "constraintName": "PK_user_connections",
         "columns": [
           {
             "columnName": "id"
@@ -729,6 +977,7 @@ module.exports = {
       "foreignKeys": [
         {
           "constraintType": "foreignKey",
+          "constraintName": "FK_user_permissions_user_id",
           "pureName": "user_permissions",
           "refTableName": "users",
           "deleteAction": "CASCADE",
@@ -743,6 +992,7 @@ module.exports = {
       "primaryKey": {
         "pureName": "user_permissions",
         "constraintType": "primaryKey",
+        "constraintName": "PK_user_permissions",
         "columns": [
           {
             "columnName": "id"
@@ -776,6 +1026,7 @@ module.exports = {
       "foreignKeys": [
         {
           "constraintType": "foreignKey",
+          "constraintName": "FK_user_roles_user_id",
           "pureName": "user_roles",
           "refTableName": "users",
           "deleteAction": "CASCADE",
@@ -788,6 +1039,7 @@ module.exports = {
         },
         {
           "constraintType": "foreignKey",
+          "constraintName": "FK_user_roles_role_id",
           "pureName": "user_roles",
           "refTableName": "roles",
           "deleteAction": "CASCADE",
@@ -802,6 +1054,7 @@ module.exports = {
       "primaryKey": {
         "pureName": "user_roles",
         "constraintType": "primaryKey",
+        "constraintName": "PK_user_roles",
         "columns": [
           {
             "columnName": "id"
@@ -815,5 +1068,6 @@ module.exports = {
   "matviews": [],
   "functions": [],
   "procedures": [],
-  "triggers": []
+  "triggers": [],
+  "schedulerEvents": []
 };

packages/api/src/utility/auditlog.js

@@ -0,0 +1,9 @@
+// only in DbGate Premium
+
+async function sendToAuditLog(req, props) {}
+async function logJsonRunnerScript(req, script) {}
+
+module.exports = {
+  sendToAuditLog,
+  logJsonRunnerScript,
+};

packages/api/src/utility/authProxy.js

@@ -40,6 +40,16 @@ function getLicenseHttpHeaders() {
   return {};
 }
 
+async function tryToGetRefreshedLicense(oldLicenseKey) {
+  return {
+    status: 'error',
+  };
+}
+
+function getAiGatewayServer() {
+  return {};
+}
+
 module.exports = {
   isAuthProxySupported,
   authProxyGetRedirectUrl,
@@ -52,4 +62,6 @@ module.exports = {
   callCompleteOnCursorApi,
   callRefactorSqlQueryApi,
   getLicenseHttpHeaders,
+  tryToGetRefreshedLicense,
+  getAiGatewayServer,
 };

packages/api/src/utility/cloudIntf.js

@@ -1,4 +1,5 @@
 const axios = require('axios');
+const crypto = require('crypto');
 const fs = require('fs-extra');
 const _ = require('lodash');
 const path = require('path');
@@ -34,11 +35,12 @@ const DBGATE_CLOUD_URL = process.env.LOCAL_DBGATE_CLOUD
   ? 'https://cloud.dbgate.udolni.net'
   : 'https://cloud.dbgate.io';
 
-async function createDbGateIdentitySession(client) {
+async function createDbGateIdentitySession(client, redirectUri) {
   const resp = await axios.default.post(
     `${DBGATE_IDENTITY_URL}/api/create-session`,
     {
       client,
+      redirectUri,
     },
     {
       headers: {
@@ -70,7 +72,7 @@ function startCloudTokenChecking(sid, callback) {
       });
       // console.log('CHECK RESP:', resp.data);
 
-      if (resp.data.email) {
+      if (resp.data?.email) {
         clearInterval(interval);
         callback(resp.data);
       }
@@ -80,6 +82,34 @@ function startCloudTokenChecking(sid, callback) {
   }, 500);
 }
 
+async function readCloudTokenHolder(sid) {
+  const resp = await axios.default.get(`${DBGATE_IDENTITY_URL}/api/get-token/${sid}`, {
+    headers: {
+      ...getLicenseHttpHeaders(),
+    },
+  });
+  if (resp.data?.email) {
+    return resp.data;
+  }
+  return null;
+}
+
+async function readCloudTestTokenHolder(email) {
+  const resp = await axios.default.post(
+    `${DBGATE_IDENTITY_URL}/api/test-token`,
+    { email },
+    {
+      headers: {
+        ...getLicenseHttpHeaders(),
+      },
+    }
+  );
+  if (resp.data?.email) {
+    return resp.data;
+  }
+  return null;
+}
+
 async function loadCloudFiles() {
   try {
     const fileContent = await fs.readFile(path.join(datadir(), 'cloud-files.jsonl'), 'utf-8');
@@ -187,7 +217,7 @@ async function updateCloudFiles(isRefresh) {
     {
       headers: {
         ...getLicenseHttpHeaders(),
-        ...(await getCloudSigninHeaders()),
+        ...(await getCloudInstanceHeaders()),
         'x-app-version': currentVersion.version,
       },
     }
@@ -271,6 +301,17 @@ async function callCloudApiGet(endpoint, signinHolder = null, additionalHeaders
   return resp.data;
 }
 
+async function getCloudInstanceHeaders() {
+  if (!(await fs.exists(path.join(datadir(), 'cloud-instance.txt')))) {
+    const newInstanceId = crypto.randomUUID();
+    await fs.writeFile(path.join(datadir(), 'cloud-instance.txt'), newInstanceId);
+  }
+  const instanceId = await fs.readFile(path.join(datadir(), 'cloud-instance.txt'), 'utf-8');
+  return {
+    'x-cloud-instance': instanceId,
+  };
+}
+
 async function callCloudApiPost(endpoint, body, signinHolder = null) {
   if (!signinHolder) {
     signinHolder = await getCloudSigninHolder();
@@ -396,4 +437,6 @@ module.exports = {
   loadCachedCloudConnection,
   putCloudContent,
   removeCloudCachedConnection,
+  readCloudTokenHolder,
+  readCloudTestTokenHolder,
 };

packages/api/src/utility/connectUtility.js

@@ -88,13 +88,33 @@ async function extractConnectionSslParams(connection) {
   return ssl;
 }
 
+async function decryptCloudConnection(connection) {
+  const { getCloudFolderEncryptor } = require('./cloudIntf');
+
+  const m = connection?._id?.match(/^cloud\:\/\/(.+)\/(.+)$/);
+  if (!m) {
+    throw new Error('Invalid cloud connection ID format');
+  }
+
+  const folid = m[1];
+  const cntid = m[2];
+
+  const folderEncryptor = await getCloudFolderEncryptor(folid);
+  return decryptConnection(connection, folderEncryptor);
+}
+
 async function connectUtility(driver, storedConnection, connectionMode, additionalOptions = null) {
   const connectionLoaded = await loadConnection(driver, storedConnection, connectionMode);
 
-  const connection = {
-    database: connectionLoaded.defaultDatabase,
-    ...decryptConnection(connectionLoaded),
-  };
+  const connection = connectionLoaded?._id?.startsWith('cloud://')
+    ? {
+        database: connectionLoaded.defaultDatabase,
+        ...(await decryptCloudConnection(connectionLoaded)),
+      }
+    : {
+        database: connectionLoaded.defaultDatabase,
+        ...decryptConnection(connectionLoaded),
+      };
 
   if (!connection.port && driver.defaultPort) {
     connection.port = driver.defaultPort.toString();

packages/api/src/utility/crypting.js

@@ -91,34 +91,36 @@ function encryptObjectPasswordField(obj, field, encryptor = null) {
   return obj;
 }
 
-function decryptObjectPasswordField(obj, field) {
+function decryptObjectPasswordField(obj, field, encryptor = null) {
   if (obj && obj[field] && obj[field].startsWith('crypt:')) {
     return {
       ...obj,
-      [field]: getInternalEncryptor().decrypt(obj[field].substring('crypt:'.length)),
+      [field]: (encryptor || getInternalEncryptor()).decrypt(obj[field].substring('crypt:'.length)),
     };
   }
   return obj;
 }
 
+const fieldsToEncrypt = ['password', 'sshPassword', 'sshKeyfilePassword', 'connectionDefinition'];
+
 function encryptConnection(connection, encryptor = null) {
   if (connection.passwordMode != 'saveRaw') {
-    connection = encryptObjectPasswordField(connection, 'password', encryptor);
-    connection = encryptObjectPasswordField(connection, 'sshPassword', encryptor);
-    connection = encryptObjectPasswordField(connection, 'sshKeyfilePassword', encryptor);
+    for (const field of fieldsToEncrypt) {
+      connection = encryptObjectPasswordField(connection, field, encryptor);
+    }
   }
   return connection;
 }
 
 function maskConnection(connection) {
   if (!connection) return connection;
-  return _.omit(connection, ['password', 'sshPassword', 'sshKeyfilePassword']);
+  return _.omit(connection, fieldsToEncrypt);
 }
 
 function decryptConnection(connection) {
-  connection = decryptObjectPasswordField(connection, 'password');
-  connection = decryptObjectPasswordField(connection, 'sshPassword');
-  connection = decryptObjectPasswordField(connection, 'sshKeyfilePassword');
+  for (const field of fieldsToEncrypt) {
+    connection = decryptObjectPasswordField(connection, field);
+  }
   return connection;
 }
 
@@ -188,9 +190,9 @@ function recryptObjectPasswordFieldInPlace(obj, field, decryptEncryptor, encrypt
 }
 
 function recryptConnection(connection, decryptEncryptor, encryptEncryptor) {
-  connection = recryptObjectPasswordField(connection, 'password', decryptEncryptor, encryptEncryptor);
-  connection = recryptObjectPasswordField(connection, 'sshPassword', decryptEncryptor, encryptEncryptor);
-  connection = recryptObjectPasswordField(connection, 'sshKeyfilePassword', decryptEncryptor, encryptEncryptor);
+  for (const field of fieldsToEncrypt) {
+    connection = recryptObjectPasswordField(connection, field, decryptEncryptor, encryptEncryptor);
+  }
   return connection;
 }
 

packages/api/src/utility/getChartExport.js

@@ -1,4 +1,12 @@
-const getChartExport = (title, config, imageFile) => {
+const getChartExport = (title, config, imageFile, plugins) => {
+  const PLUGIN_TAGS = {
+    zoom: '<script src="https://cdnjs.cloudflare.com/ajax/libs/chartjs-plugin-zoom/1.2.0/chartjs-plugin-zoom.min.js" integrity="sha512-TT0wAMqqtjXVzpc48sI0G84rBP+oTkBZPgeRYIOVRGUdwJsyS3WPipsNh///ay2LJ+onCM23tipnz6EvEy2/UA==" crossorigin="anonymous" referrerpolicy="no-referrer"></script>',
+    dataLabels:
+      '<script src="https://cdnjs.cloudflare.com/ajax/libs/chartjs-plugin-datalabels/2.2.0/chartjs-plugin-datalabels.min.js" integrity="sha512-JPcRR8yFa8mmCsfrw4TNte1ZvF1e3+1SdGMslZvmrzDYxS69J7J49vkFL8u6u8PlPJK+H3voElBtUCzaXj+6ig==" crossorigin="anonymous" referrerpolicy="no-referrer"></script>',
+    outlabels:
+      '<script src="https://cdn.jsdelivr.net/npm/@energiency/chartjs-plugin-piechart-outlabels@1.3.4/dist/chartjs-plugin-piechart-outlabels.min.js"></script>',
+  };
+
   return `<html>
 <meta charset='utf-8'>
 
@@ -8,7 +16,7 @@ const getChartExport = (title, config, imageFile) => {
     <script src="https://cdnjs.cloudflare.com/ajax/libs/moment.js/2.29.1/moment.min.js" integrity="sha512-qTXRIMyZIFb8iQcfjXWCO8+M5Tbc38Qi5WzdPOYZHIlZpzBHG3L3by84BBBOiRGiEb7KKtAOAs5qYdUiZiQNNQ==" crossorigin="anonymous" referrerpolicy="no-referrer"></script>
     <script src="https://cdnjs.cloudflare.com/ajax/libs/chartjs-adapter-moment/1.0.0/chartjs-adapter-moment.min.js" integrity="sha512-oh5t+CdSBsaVVAvxcZKy3XJdP7ZbYUBSRCXDTVn0ODewMDDNnELsrG9eDm8rVZAQg7RsDD/8K3MjPAFB13o6eA==" crossorigin="anonymous" referrerpolicy="no-referrer"></script>
     <script src="https://cdnjs.cloudflare.com/ajax/libs/hammer.js/2.0.8/hammer.min.js" integrity="sha512-UXumZrZNiOwnTcZSHLOfcTs0aos2MzBWHXOHOuB0J/R44QB0dwY5JgfbvljXcklVf65Gc4El6RjZ+lnwd2az2g==" crossorigin="anonymous" referrerpolicy="no-referrer"></script>
-    <script src="https://cdnjs.cloudflare.com/ajax/libs/chartjs-plugin-zoom/1.2.0/chartjs-plugin-zoom.min.js" integrity="sha512-TT0wAMqqtjXVzpc48sI0G84rBP+oTkBZPgeRYIOVRGUdwJsyS3WPipsNh///ay2LJ+onCM23tipnz6EvEy2/UA==" crossorigin="anonymous" referrerpolicy="no-referrer"></script>
+    ${plugins.map(plugin => PLUGIN_TAGS[plugin] ?? '')}
 
     <style>
         a { text-decoration: none }
@@ -45,7 +53,7 @@ const getChartExport = (title, config, imageFile) => {
     </div>
 
     <div class="footer">
-        Exported from <a href='https://dbgate.org/' target='_blank'>DbGate</a>, powered by <a href='https://www.chartjs.org/' target='_blank'>Chart.js</a>
+        Exported from <a href='https://dbgate.io/' target='_blank'>DbGate</a>, powered by <a href='https://www.chartjs.org/' target='_blank'>Chart.js</a>
     </div>
 </body>
 

packages/api/src/utility/getMapExport.js

@@ -18,7 +18,7 @@ const getMapExport = (geoJson) => {
     leaflet
       .tileLayer('https://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png', {
         maxZoom: 19,
-        attribution: '<a href="https://dbgate.org" title="Exported from DbGate">DbGate</a> | © OpenStreetMap',
+        attribution: '<a href="https://dbgate.io" title="Exported from DbGate">DbGate</a> | © OpenStreetMap',
       })
       .addTo(map);
       

packages/api/src/utility/handleQueryStream.js

@@ -14,13 +14,9 @@ class QueryStreamTableWriter {
     this.currentChangeIndex = 1;
     this.initializedFile = false;
     this.sesid = sesid;
-    // if (isProApp()) {
-    //   this.chartProcessor = new ChartProcessor();
-    // }
-    this.chartProcessor = new ChartProcessor();
   }
 
-  initializeFromQuery(structure, resultIndex, chartDefinition) {
+  initializeFromQuery(structure, resultIndex, chartDefinition, autoDetectCharts = false) {
     this.jslid = crypto.randomUUID();
     this.currentFile = path.join(jsldir(), `${this.jslid}.jsonl`);
     fs.writeFileSync(
@@ -34,8 +30,8 @@ class QueryStreamTableWriter {
     this.writeCurrentStats(false, false);
     this.resultIndex = resultIndex;
     this.initializedFile = true;
-    if (isProApp() && chartDefinition) {
-      this.chartProcessor = new ChartProcessor([chartDefinition]);
+    if (isProApp() && (chartDefinition || autoDetectCharts)) {
+      this.chartProcessor = chartDefinition ? new ChartProcessor([chartDefinition]) : new ChartProcessor();
     }
     process.send({ msgtype: 'recordset', jslid: this.jslid, resultIndex, sesid: this.sesid });
   }
@@ -138,12 +134,14 @@ class StreamHandler {
     startLine,
     sesid = undefined,
     limitRows = undefined,
-    frontMatter = undefined
+    frontMatter = undefined,
+    autoDetectCharts = false
   ) {
     this.recordset = this.recordset.bind(this);
     this.startLine = startLine;
     this.sesid = sesid;
     this.frontMatter = frontMatter;
+    this.autoDetectCharts = autoDetectCharts;
     this.limitRows = limitRows;
     this.rowsLimitOverflow = false;
     this.row = this.row.bind(this);
@@ -177,7 +175,8 @@ class StreamHandler {
     this.currentWriter.initializeFromQuery(
       Array.isArray(columns) ? { columns } : columns,
       this.queryStreamInfoHolder.resultIndex,
-      this.frontMatter?.[`chart-${this.queryStreamInfoHolder.resultIndex + 1}`]
+      this.frontMatter?.[`chart-${this.queryStreamInfoHolder.resultIndex + 1}`],
+      this.autoDetectCharts
     );
     this.queryStreamInfoHolder.resultIndex += 1;
     this.rowCounter = 0;
@@ -252,7 +251,8 @@ function handleQueryStream(
   sqlItem,
   sesid = undefined,
   limitRows = undefined,
-  frontMatter = undefined
+  frontMatter = undefined,
+  autoDetectCharts = false
 ) {
   return new Promise((resolve, reject) => {
     const start = sqlItem.trimStart || sqlItem.start;
@@ -262,7 +262,8 @@ function handleQueryStream(
       start && start.line,
       sesid,
       limitRows,
-      frontMatter
+      frontMatter,
+      autoDetectCharts
     );
     driver.stream(dbhan, sqlItem.text, handler);
   });

packages/api/src/utility/loginchecker.js

@@ -0,0 +1,18 @@
+// only in DbGate Premium
+
+function markUserAsActive(licenseUid, token) {}
+
+async function isLoginLicensed(req, licenseUid) {
+  return true;
+}
+
+function markLoginAsLoggedOut(licenseUid) {}
+
+const LOGIN_LIMIT_ERROR = '';
+
+module.exports = {
+  markUserAsActive,
+  isLoginLicensed,
+  markLoginAsLoggedOut,
+  LOGIN_LIMIT_ERROR,
+};

packages/datalib/src/CollectionGridDisplay.ts

@@ -70,6 +70,7 @@ function getDisplayColumn(basePath, columnName, display: CollectionGridDisplay)
     isPartitionKey: !!display?.collection?.partitionKey?.find(x => x.columnName == uniqueName),
     isClusterKey: !!display?.collection?.clusterKey?.find(x => x.columnName == uniqueName),
     isUniqueKey: !!display?.collection?.uniqueKey?.find(x => x.columnName == uniqueName),
+    hasAutoValue: !!display?.collection?.autoValueColumns?.find(x => x.columnName == uniqueName),
   };
 }
 
@@ -101,9 +102,10 @@ export class CollectionGridDisplay extends GridDisplay {
     setCache: ChangeCacheFunc,
     loadedRows,
     changeSet,
-    readOnly = false
+    readOnly = false,
+    currentSettings = null
   ) {
-    super(config, setConfig, cache, setCache, driver);
+    super(config, setConfig, cache, setCache, driver, undefined, undefined, currentSettings);
     const changedDocs = _.compact(changeSet.updates.map(chs => chs.document));
     const insertedDocs = _.compact(changeSet.inserts.map(chs => chs.fields));
     this.columns = analyseCollectionDisplayColumns([...(loadedRows || []), ...changedDocs, ...insertedDocs], this);

packages/datalib/src/JslGridDisplay.ts

@@ -17,9 +17,10 @@ export class JslGridDisplay extends GridDisplay {
     isDynamicStructure: boolean,
     supportsReload: boolean,
     editable: boolean = false,
-    driver: EngineDriver = null
+    driver: EngineDriver = null,
+    currentSettings = null
   ) {
-    super(config, setConfig, cache, setCache, driver);
+    super(config, setConfig, cache, setCache, driver, undefined, undefined, currentSettings);
 
     this.filterable = true;
     this.sortable = true;

packages/datalib/src/PerspectiveDataLoader.ts

@@ -106,6 +106,7 @@ export class PerspectiveDataLoader {
       conid: props.databaseConfig.conid,
       database: props.databaseConfig.database,
       select,
+      auditLogSessionGroup: 'perspective',
     });
 
     if (response.errorMessage) return response;
@@ -152,6 +153,7 @@ export class PerspectiveDataLoader {
         pureName,
         aggregate,
       },
+      auditLogSessionGroup: 'perspective',
     });
 
     if (response.errorMessage) return response;
@@ -227,6 +229,7 @@ export class PerspectiveDataLoader {
       conid: props.databaseConfig.conid,
       database: props.databaseConfig.database,
       select,
+      auditLogSessionGroup: 'perspective',
     });
 
     if (response.errorMessage) return response;
@@ -284,6 +287,7 @@ export class PerspectiveDataLoader {
       conid: props.databaseConfig.conid,
       database: props.databaseConfig.database,
       options,
+      auditLogSessionGroup: 'perspective',
     });
 
     if (response.errorMessage) return response;
@@ -330,6 +334,7 @@ export class PerspectiveDataLoader {
       conid: props.databaseConfig.conid,
       database: props.databaseConfig.database,
       select,
+      auditLogSessionGroup: 'perspective',
     });
 
     if (response.errorMessage) return response;
@@ -356,6 +361,7 @@ export class PerspectiveDataLoader {
       conid: props.databaseConfig.conid,
       database: props.databaseConfig.database,
       options,
+      auditLogSessionGroup: 'perspective',
     });
 
     return response;

packages/datalib/src/ViewGridDisplay.ts

@@ -12,9 +12,10 @@ export class ViewGridDisplay extends GridDisplay {
     cache: GridCache,
     setCache: ChangeCacheFunc,
     dbinfo: DatabaseInfo,
-    serverVersion
+    serverVersion,
+    currentSettings
   ) {
-    super(config, setConfig, cache, setCache, driver, dbinfo, serverVersion);
+    super(config, setConfig, cache, setCache, driver, dbinfo, serverVersion, currentSettings);
     this.columns = this.getDisplayColumns(view);
     this.formColumns = this.columns;
     this.filterable = true;

packages/datalib/src/chartDefinitions.ts

@@ -1,4 +1,4 @@
-export type ChartTypeEnum = 'bar' | 'line' | 'pie' | 'polarArea';
+export type ChartTypeEnum = 'bar' | 'line' | 'timeline' | 'pie' | 'polarArea';
 export type ChartXTransformFunction =
   | 'identity'
   | 'date:minute'
@@ -17,13 +17,17 @@ export const ChartConstDefaults = {
 };
 
 export const ChartLimits = {
-  AUTODETECT_CHART_LIMIT: 10, // limit for auto-detecting charts, to avoid too many charts
+  AUTODETECT_CHART_LIMIT: 10, // limit for auto-detecting charts, to avoid too many charts (after APPLY_LIMIT_AFTER_ROWS rows)
+  AUTODETECT_CHART_TOTAL_LIMIT: 32, // limit for auto-detecting charts, to avoid too many charts (for first APPLY_LIMIT_AFTER_ROWS rows)
   AUTODETECT_MEASURES_LIMIT: 10, // limit for auto-detecting measures, to avoid too many measures
   APPLY_LIMIT_AFTER_ROWS: 100,
   MAX_DISTINCT_VALUES: 10, // max number of distinct values to keep in topDistinctValues
   VALID_VALUE_RATIO_LIMIT: 0.5, // limit for valid value ratio, y defs below this will not be used in auto-detect
   PIE_RATIO_LIMIT: 0.05, // limit for other values in pie chart, if the value is below this, it will be grouped into "Other"
   PIE_COUNT_LIMIT: 10, // limit for number of pie chart slices, if the number of slices is above this, it will be grouped into "Other"
+  MAX_PIE_COUNT_LIMIT: 50, // max pie limit
+  CHART_FILL_LIMIT: 10000, // limit for filled charts (time intervals), to avoid too many points
+  CHART_GROUP_LIMIT: 32, // limit for number of groups in a chart
 };
 
 export interface ChartXFieldDefinition {
@@ -47,9 +51,12 @@ export interface ChartDefinition {
   title?: string;
   pieRatioLimit?: number; // limit for pie chart, if the value is below this, it will be grouped into "Other"
   pieCountLimit?: number; // limit for number of pie chart slices, if the number of slices is above this, it will be grouped into "Other"
+  trimXCountLimit?: number; // limit for number of x values, if the number of x values is above this, it will be trimmed
 
   xdef: ChartXFieldDefinition;
   ydefs: ChartYFieldDefinition[];
+  groupingField?: string;
+  groupTransformFunction?: ChartXTransformFunction;
 
   useDataLabels?: boolean;
   dataLabelFormatter?: ChartDataLabelFormatter;
@@ -67,6 +74,7 @@ export interface ChartDateParsed {
 
 export interface ChartAvailableColumn {
   field: string;
+  dataType: 'none' | 'string' | 'number' | 'date' | 'mixed';
 }
 
 export interface ProcessedChart {
@@ -75,14 +83,18 @@ export interface ProcessedChart {
   rowsAdded: number;
   buckets: { [key: string]: any }; // key is the bucket key, value is aggregated data
   bucketKeysOrdered: string[];
-  bucketKeyDateParsed: { [key: string]: ChartDateParsed }; // key is the bucket key, value is parsed date
+  bucketKeysSet: Set<string>;
+  bucketKeyDateParsed: { [key: string]: ChartDateParsed }; // key is the bucket key (without group::), value is parsed date
   isGivenDefinition: boolean; // true if the chart was created with a given definition, false if it was created from raw data
   invalidXRows: number;
   invalidYRows: { [key: string]: number }; // key is the y field, value is the count of invalid rows
   validYRows: { [key: string]: number }; // key is the field, value is the count of valid rows
+  groups: string[];
+  groupSet: Set<string>;
 
   topDistinctValues: { [key: string]: Set<any> }; // key is the field, value is the set of distinct values
   availableColumns: ChartAvailableColumn[];
+  errorMessage?: string; // error message if there was an error processing the chart
 
   definition: ChartDefinition;
 }

packages/datalib/src/chartProcessor.ts

@@ -3,16 +3,23 @@ import {
   ChartDateParsed,
   ChartDefinition,
   ChartLimits,
+  ChartYFieldDefinition,
   ProcessedChart,
 } from './chartDefinitions';
 import _sortBy from 'lodash/sortBy';
 import _sum from 'lodash/sum';
+import _zipObject from 'lodash/zipObject';
+import _mapValues from 'lodash/mapValues';
+import _pick from 'lodash/pick';
 import {
   aggregateChartNumericValuesFromSource,
   autoAggregateCompactTimelineChart,
+  chartsHaveSimilarRange,
   computeChartBucketCardinality,
   computeChartBucketKey,
   fillChartTimelineBuckets,
+  getChartYRange,
+  runTransformFunction,
   tryParseChartDate,
 } from './chartTools';
 import { getChartScore, getChartYFieldScore } from './chartScoring';
@@ -24,6 +31,7 @@ export class ChartProcessor {
   availableColumns: ChartAvailableColumn[] = [];
   autoDetectCharts = false;
   rowsAdded = 0;
+  errorMessage?: string;
 
   constructor(public givenDefinitions: ChartDefinition[] = []) {
     for (const definition of givenDefinitions) {
@@ -39,6 +47,9 @@ export class ChartProcessor {
         availableColumns: [],
         validYRows: {},
         topDistinctValues: {},
+        groups: [],
+        groupSet: new Set<string>(),
+        bucketKeysSet: new Set<string>(),
       });
     }
     this.autoDetectCharts = this.givenDefinitions.length == 0;
@@ -67,6 +78,91 @@ export class ChartProcessor {
   //   this.chartsBySignature[signature] = chart;
   //   return chart;
   // }
+  runAutoDetectCharts(
+    dateColumns: { [key: string]: ChartDateParsed },
+    numericColumnsForAutodetect: { [key: string]: number },
+    stringColumns: { [key: string]: string }
+  ) {
+    const processColumnType = (columns, transformTest, chartType, transformFunction) => {
+      for (const xcol in columns) {
+        for (const groupingField of [undefined, ...Object.keys(stringColumns)]) {
+          if (xcol == groupingField) {
+            continue;
+          }
+
+          let usedChart = this.chartsProcessing.find(
+            chart =>
+              !chart.isGivenDefinition &&
+              chart.definition.xdef.field === xcol &&
+              transformTest(chart.definition.xdef.transformFunction) &&
+              chart.definition.groupingField == groupingField
+          );
+
+          if (
+            !usedChart &&
+            (this.rowsAdded < ChartLimits.APPLY_LIMIT_AFTER_ROWS ||
+              this.chartsProcessing.length < ChartLimits.AUTODETECT_CHART_LIMIT)
+          ) {
+            usedChart = {
+              definition: {
+                chartType,
+                xdef: {
+                  field: xcol,
+                  transformFunction,
+                },
+                ydefs: [
+                  {
+                    field: '__count',
+                    aggregateFunction: 'count',
+                  },
+                ],
+                groupingField,
+              },
+              rowsAdded: 0,
+              bucketKeysOrdered: [],
+              buckets: {},
+              groups: [],
+              bucketKeyDateParsed: {},
+              isGivenDefinition: false,
+              invalidXRows: 0,
+              invalidYRows: {},
+              availableColumns: [],
+              validYRows: {},
+              topDistinctValues: {},
+              groupSet: new Set<string>(),
+              bucketKeysSet: new Set<string>(),
+            };
+            this.chartsProcessing.push(usedChart);
+          }
+
+          if (!usedChart) {
+            continue; // chart not created - probably too many charts already
+          }
+
+          for (const [key, value] of Object.entries(numericColumnsForAutodetect)) {
+            // if (value == null) continue;
+            // if (key == datecol) continue; // skip date column itself
+
+            const existingYDef = usedChart.definition.ydefs.find(y => y.field === key);
+            if (
+              !existingYDef &&
+              (this.rowsAdded < ChartLimits.APPLY_LIMIT_AFTER_ROWS ||
+                usedChart.definition.ydefs.length < ChartLimits.AUTODETECT_MEASURES_LIMIT)
+            ) {
+              const newYDef: ChartYFieldDefinition = {
+                field: key,
+                aggregateFunction: 'sum',
+              };
+              usedChart.definition.ydefs.push(newYDef);
+            }
+          }
+        }
+      }
+    };
+
+    processColumnType(dateColumns, transform => transform?.startsWith('date:'), 'timeline', 'date:day');
+    processColumnType(stringColumns, transform => transform == 'identity', 'bar', 'identity');
+  }
 
   addRow(row: any) {
     const dateColumns: { [key: string]: ChartDateParsed } = {};
@@ -76,9 +172,14 @@ export class ChartProcessor {
 
     for (const [key, value] of Object.entries(row)) {
       const number: number = typeof value == 'string' ? Number(value) : typeof value == 'number' ? value : NaN;
-      this.availableColumnsDict[key] = {
-        field: key,
-      };
+      let availableColumn = this.availableColumnsDict[key];
+      if (!availableColumn) {
+        availableColumn = {
+          field: key,
+          dataType: 'none',
+        };
+        this.availableColumnsDict[key] = availableColumn;
+      }
 
       const keyLower = key.toLowerCase();
       const keyIsId = keyLower.endsWith('_id') || keyLower == 'id' || key.endsWith('Id');
@@ -86,6 +187,12 @@ export class ChartProcessor {
       const parsedDate = tryParseChartDate(value);
       if (parsedDate) {
         dateColumns[key] = parsedDate;
+        if (availableColumn.dataType == 'none') {
+          availableColumn.dataType = 'date';
+        }
+        if (availableColumn.dataType != 'date') {
+          availableColumn.dataType = 'mixed';
+        }
         continue;
       }
 
@@ -94,85 +201,52 @@ export class ChartProcessor {
         if (!keyIsId) {
           numericColumnsForAutodetect[key] = number; // for auto-detecting charts
         }
+        if (availableColumn.dataType == 'none') {
+          availableColumn.dataType = 'number';
+        }
+        if (availableColumn.dataType != 'number') {
+          availableColumn.dataType = 'mixed';
+        }
         continue;
       }
 
       if (typeof value === 'string' && isNaN(number) && value.length < 100) {
         stringColumns[key] = value;
+        if (availableColumn.dataType == 'none') {
+          availableColumn.dataType = 'string';
+        }
+        if (availableColumn.dataType != 'string') {
+          availableColumn.dataType = 'mixed';
+        }
       }
     }
 
     // const sortedNumericColumnns = Object.keys(numericColumns).sort();
 
     if (this.autoDetectCharts) {
-      // create charts from data, if there are no given definitions
-      for (const datecol in dateColumns) {
-        let usedChart = this.chartsProcessing.find(
-          chart =>
-            !chart.isGivenDefinition &&
-            chart.definition.xdef.field === datecol &&
-            chart.definition.xdef.transformFunction?.startsWith('date:')
-        );
-
-        if (
-          !usedChart &&
-          (this.rowsAdded < ChartLimits.APPLY_LIMIT_AFTER_ROWS ||
-            this.chartsProcessing.length < ChartLimits.AUTODETECT_CHART_LIMIT)
-        ) {
-          usedChart = {
-            definition: {
-              chartType: 'line',
-              xdef: {
-                field: datecol,
-                transformFunction: 'date:day',
-              },
-              ydefs: [],
-            },
-            rowsAdded: 0,
-            bucketKeysOrdered: [],
-            buckets: {},
-            bucketKeyDateParsed: {},
-            isGivenDefinition: false,
-            invalidXRows: 0,
-            invalidYRows: {},
-            availableColumns: [],
-            validYRows: {},
-            topDistinctValues: {},
-          };
-          this.chartsProcessing.push(usedChart);
-        }
-
-        for (const [key, value] of Object.entries(row)) {
-          if (value == null) continue;
-          if (key == datecol) continue; // skip date column itself
-          let existingYDef = usedChart.definition.ydefs.find(y => y.field === key);
-          if (
-            !existingYDef &&
-            (this.rowsAdded < ChartLimits.APPLY_LIMIT_AFTER_ROWS ||
-              usedChart.definition.ydefs.length < ChartLimits.AUTODETECT_MEASURES_LIMIT)
-          ) {
-            existingYDef = {
-              field: key,
-              aggregateFunction: 'sum',
-            };
-            usedChart.definition.ydefs.push(existingYDef);
-          }
-        }
-      }
+      this.runAutoDetectCharts(dateColumns, numericColumnsForAutodetect, stringColumns);
     }
 
     // apply on all charts with this date column
     for (const chart of this.chartsProcessing) {
-      this.applyRawData(
-        chart,
-        row,
-        dateColumns[chart.definition.xdef.field],
-        chart.isGivenDefinition ? numericColumns : numericColumnsForAutodetect,
-        stringColumns
-      );
+      if (chart.errorMessage) {
+        continue; // skip charts with errors
+      }
+
+      this.applyRawData(chart, row, dateColumns[chart.definition.xdef.field], numericColumns, stringColumns);
+
+      if (Object.keys(chart.buckets).length > ChartLimits.CHART_FILL_LIMIT) {
+        chart.errorMessage = `Chart has too many buckets, limit is ${ChartLimits.CHART_FILL_LIMIT}.`;
+      }
     }
 
     for (let i = 0; i < this.chartsProcessing.length; i++) {
+      if (this.chartsProcessing[i].errorMessage) {
+        continue; // skip charts with errors
+      }
+      if (this.chartsProcessing[i].definition.chartType != 'timeline') {
+        continue; // skip non-timeline charts
+      }
       this.chartsProcessing[i] = autoAggregateCompactTimelineChart(this.chartsProcessing[i]);
     }
 
@@ -210,30 +284,79 @@ export class ChartProcessor {
     }
   }
 
+  splitChartsByYDefs() {
+    const newCharts: ProcessedChart[] = [];
+
+    for (const chart of this.chartsProcessing) {
+      if (chart.isGivenDefinition) {
+        newCharts.push(chart);
+        continue;
+      }
+      const yRanges = chart.definition.ydefs.map(ydef => getChartYRange(chart, ydef).max);
+      const yRangeByField = _zipObject(
+        chart.definition.ydefs.map(ydef => ydef.field),
+        yRanges
+      );
+      let ydefsToAssign = chart.definition.ydefs.map(ydef => ydef.field);
+      while (ydefsToAssign.length > 0) {
+        const first = ydefsToAssign.shift();
+        const additionals = [];
+        for (const candidate of ydefsToAssign) {
+          if (chartsHaveSimilarRange(yRangeByField[first], yRangeByField[candidate])) {
+            additionals.push(candidate);
+          }
+        }
+
+        const ydefsCurrent = [first, ...additionals];
+        const partialChart: ProcessedChart = {
+          ...chart,
+          definition: {
+            ...chart.definition,
+            ydefs: ydefsCurrent.map(y => chart.definition.ydefs.find(yd => yd.field === y) as ChartYFieldDefinition),
+          },
+          buckets: _mapValues(chart.buckets, bucket => _pick(bucket, ydefsCurrent)),
+        };
+
+        newCharts.push(partialChart);
+        ydefsToAssign = ydefsToAssign.filter(y => !additionals.includes(y));
+      }
+    }
+    this.chartsProcessing = newCharts;
+  }
+
   finalize() {
+    this.splitChartsByYDefs();
     this.applyLimitsOnCharts();
     this.availableColumns = Object.values(this.availableColumnsDict);
     for (const chart of this.chartsProcessing) {
+      if (chart.errorMessage) {
+        this.charts.push({ ...chart, availableColumns: this.availableColumns });
+        continue;
+      }
       let addedChart: ProcessedChart = chart;
-      if (chart.rowsAdded == 0) {
+      if (chart.rowsAdded == 0 && !chart.isGivenDefinition) {
         continue; // skip empty charts
       }
       const sortOrder = chart.definition.xdef.sortOrder ?? 'ascKeys';
       if (sortOrder != 'natural') {
         if (sortOrder == 'ascKeys' || sortOrder == 'descKeys') {
-          if (chart.definition.xdef.transformFunction.startsWith('date:')) {
+          if (chart.definition.chartType == 'timeline' && chart.definition.xdef.transformFunction.startsWith('date:')) {
             addedChart = autoAggregateCompactTimelineChart(addedChart);
             fillChartTimelineBuckets(addedChart);
           }
 
-          addedChart.bucketKeysOrdered = _sortBy(Object.keys(addedChart.buckets));
+          if (addedChart.errorMessage) {
+            this.charts.push(addedChart);
+            continue;
+          }
+          addedChart.bucketKeysOrdered = _sortBy([...addedChart.bucketKeysSet]);
           if (sortOrder == 'descKeys') {
             addedChart.bucketKeysOrdered.reverse();
           }
         }
 
         if (sortOrder == 'ascValues' || sortOrder == 'descValues') {
-          addedChart.bucketKeysOrdered = _sortBy(Object.keys(addedChart.buckets), key =>
+          addedChart.bucketKeysOrdered = _sortBy([...addedChart.bucketKeysSet], key =>
             computeChartBucketCardinality(addedChart.buckets[key])
           );
           if (sortOrder == 'descValues') {
@@ -256,31 +379,45 @@ export class ChartProcessor {
         };
       }
 
+      if (
+        addedChart.definition.trimXCountLimit != null &&
+        addedChart.bucketKeysOrdered.length > addedChart.definition.trimXCountLimit
+      ) {
+        addedChart.bucketKeysOrdered = addedChart.bucketKeysOrdered.slice(0, addedChart.definition.trimXCountLimit);
+      }
+
       if (addedChart) {
         addedChart.availableColumns = this.availableColumns;
         this.charts.push(addedChart);
       }
 
       this.groupPieOtherBuckets(addedChart);
+
+      addedChart.groups = [...addedChart.groupSet];
+      addedChart.bucketKeysSet = undefined;
+      addedChart.groupSet = undefined;
     }
 
     this.charts = [
       ...this.charts.filter(x => x.isGivenDefinition),
       ..._sortBy(
-        this.charts.filter(x => !x.isGivenDefinition),
+        this.charts.filter(x => !x.isGivenDefinition && !x.errorMessage && x.definition.ydefs.length > 0),
         chart => -getChartScore(chart)
       ),
     ];
   }
   groupPieOtherBuckets(chart: ProcessedChart) {
-    if (chart.definition.chartType !== 'pie') {
+    if (chart.definition.chartType != 'pie' && chart.definition.chartType != 'polarArea') {
       return; // only for pie charts
     }
     const ratioLimit = chart.definition.pieRatioLimit ?? ChartLimits.PIE_RATIO_LIMIT;
-    const countLimit = chart.definition.pieCountLimit ?? ChartLimits.PIE_COUNT_LIMIT;
-    if (ratioLimit == 0 && countLimit == 0) {
-      return; // no grouping if limit is 0
+    let countLimit = chart.definition.pieCountLimit ?? ChartLimits.PIE_COUNT_LIMIT;
+    if (!countLimit || countLimit < 1 || countLimit > ChartLimits.MAX_PIE_COUNT_LIMIT) {
+      countLimit = ChartLimits.MAX_PIE_COUNT_LIMIT; // limit to max pie count
     }
+    // if (ratioLimit == 0 && countLimit == 0) {
+    //   return; // no grouping if limit is 0
+    // }
     const otherBucket: any = {};
     let newBuckets: any = {};
     const cardSum = _sum(Object.values(chart.buckets).map(bucket => computeChartBucketCardinality(bucket)));
@@ -345,6 +482,15 @@ export class ChartProcessor {
     }
 
     const [bucketKey, bucketKeyParsed] = computeChartBucketKey(dateParsed, chart, row);
+    const bucketGroup = chart.definition.groupingField
+      ? runTransformFunction(row[chart.definition.groupingField], chart.definition.groupTransformFunction)
+      : null;
+    if (bucketGroup) {
+      chart.groupSet.add(bucketGroup);
+    }
+    if (chart.groupSet.size > ChartLimits.CHART_GROUP_LIMIT) {
+      chart.errorMessage = `Chart has too many groups, limit is ${ChartLimits.CHART_GROUP_LIMIT}.`;
+    }
 
     if (!bucketKey) {
       return; // skip if no bucket key
@@ -361,14 +507,19 @@ export class ChartProcessor {
       chart.maxX = bucketKey;
     }
 
-    if (!chart.buckets[bucketKey]) {
-      chart.buckets[bucketKey] = {};
+    const groupedBucketKey = chart.definition.groupingField ? `${bucketGroup ?? ''}::${bucketKey}` : bucketKey;
+    if (!chart.buckets[groupedBucketKey]) {
+      chart.buckets[groupedBucketKey] = {};
+    }
+
+    if (!chart.bucketKeysSet.has(bucketKey)) {
+      chart.bucketKeysSet.add(bucketKey);
       if (chart.definition.xdef.sortOrder == 'natural') {
         chart.bucketKeysOrdered.push(bucketKey);
       }
     }
 
-    aggregateChartNumericValuesFromSource(chart, bucketKey, numericColumns, row);
+    aggregateChartNumericValuesFromSource(chart, groupedBucketKey, numericColumns, row);
     chart.rowsAdded += 1;
   }
 }

packages/datalib/src/chartScoring.ts

@@ -3,12 +3,23 @@ import _sum from 'lodash/sum';
 import { ChartLimits, ChartYFieldDefinition, ProcessedChart } from './chartDefinitions';
 
 export function getChartScore(chart: ProcessedChart): number {
+  if (chart.errorMessage) {
+    return -1; // negative score for charts with errors
+  }
   let res = 0;
   res += chart.rowsAdded * 5;
 
   const ydefScores = chart.definition.ydefs.map(yField => getChartYFieldScore(chart, yField));
   const sorted = _sortBy(ydefScores).reverse();
   res += _sum(sorted.slice(0, ChartLimits.AUTODETECT_MEASURES_LIMIT));
+
+  if (chart.groupSet?.size >= 2 && chart.groupSet?.size <= 6) {
+    res += 50; // bonus for nice grouping
+  }
+  if (chart.groupSet?.size == 1) {
+    res -= 20; // penalty for single group
+  }
+
   return res;
 }
 

packages/datalib/src/chartTools.ts

@@ -3,15 +3,19 @@ import _sumBy from 'lodash/sumBy';
 import {
   ChartConstDefaults,
   ChartDateParsed,
+  ChartDefinition,
   ChartLimits,
   ChartXTransformFunction,
+  ChartYFieldDefinition,
   ProcessedChart,
 } from './chartDefinitions';
 import { addMinutes, addHours, addDays, addMonths, addYears } from 'date-fns';
 
 export function getChartDebugPrint(chart: ProcessedChart) {
   let res = '';
-  res += `Chart: ${chart.definition.chartType} (${chart.definition.xdef.transformFunction})\n`;
+  res += `Chart: ${chart.definition.chartType} (${chart.definition.xdef.transformFunction}): (${chart.definition.ydefs
+    .map(yd => yd.field)
+    .join(', ')})\n`;
   for (const key of chart.bucketKeysOrdered) {
     res += `${key}: ${_toPairs(chart.buckets[key])
       .map(([k, v]) => `${k}=${v}`)
@@ -34,22 +38,53 @@ export function tryParseChartDate(dateInput: any): ChartDateParsed | null {
   }
 
   if (typeof dateInput !== 'string') return null;
-  const m = dateInput.match(
+  const dateMatch = dateInput.match(
     /^(\d{4})-(\d{2})-(\d{2})(?:[ T](\d{2}):(\d{2}):(\d{2})(?:\.(\d+))?(Z|[+-]\d{2}:\d{2})?)?$/
   );
-  if (!m) return null;
+  const monthMatch = dateInput.match(/^(\d{4})-(\d{2})$/);
+  // const yearMatch = dateInput.match(/^(\d{4})$/);
 
-  const [_notUsed, year, month, day, hour, minute, second, fraction] = m;
+  if (dateMatch) {
+    const [_notUsed, year, month, day, hour, minute, second, fraction] = dateMatch;
 
-  return {
-    year: parseInt(year, 10),
-    month: parseInt(month, 10),
-    day: parseInt(day, 10),
-    hour: parseInt(hour, 10) || 0,
-    minute: parseInt(minute, 10) || 0,
-    second: parseInt(second, 10) || 0,
-    fraction: fraction || undefined,
-  };
+    return {
+      year: parseInt(year, 10),
+      month: parseInt(month, 10),
+      day: parseInt(day, 10),
+      hour: parseInt(hour, 10) || 0,
+      minute: parseInt(minute, 10) || 0,
+      second: parseInt(second, 10) || 0,
+      fraction: fraction || undefined,
+    };
+  }
+
+  if (monthMatch) {
+    const [_notUsed, year, month] = monthMatch;
+    return {
+      year: parseInt(year, 10),
+      month: parseInt(month, 10),
+      day: 1,
+      hour: 0,
+      minute: 0,
+      second: 0,
+      fraction: undefined,
+    };
+  }
+
+  // if (yearMatch) {
+  //   const [_notUsed, year] = yearMatch;
+  //   return {
+  //     year: parseInt(year, 10),
+  //     month: 1,
+  //     day: 1,
+  //     hour: 0,
+  //     minute: 0,
+  //     second: 0,
+  //     fraction: undefined,
+  //   };
+  // }
+
+  return null;
 }
 
 function pad2Digits(number) {
@@ -133,6 +168,33 @@ export function incrementChartDate(value: ChartDateParsed, transform: ChartXTran
   }
 }
 
+export function runTransformFunction(value: string, transformFunction: ChartXTransformFunction): string {
+  const dateParsed = tryParseChartDate(value);
+  switch (transformFunction) {
+    case 'date:year':
+      return dateParsed ? `${dateParsed.year}` : null;
+    case 'date:month':
+      return dateParsed ? `${dateParsed.year}-${pad2Digits(dateParsed.month)}` : null;
+    case 'date:day':
+      return dateParsed ? `${dateParsed.year}-${pad2Digits(dateParsed.month)}-${pad2Digits(dateParsed.day)}` : null;
+    case 'date:hour':
+      return dateParsed
+        ? `${dateParsed.year}-${pad2Digits(dateParsed.month)}-${pad2Digits(dateParsed.day)} ${pad2Digits(
+            dateParsed.hour
+          )}`
+        : null;
+    case 'date:minute':
+      return dateParsed
+        ? `${dateParsed.year}-${pad2Digits(dateParsed.month)}-${pad2Digits(dateParsed.day)} ${pad2Digits(
+            dateParsed.hour
+          )}:${pad2Digits(dateParsed.minute)}`
+        : null;
+    case 'identity':
+    default:
+      return value;
+  }
+}
+
 export function computeChartBucketKey(
   dateParsed: ChartDateParsed,
   chart: ProcessedChart,
@@ -268,7 +330,27 @@ export function compareChartDatesParsed(
   }
 }
 
-function getParentDateBucketKey(bucketKey: string, transform: ChartXTransformFunction): string | null {
+function extractBucketKeyWithoutGroup(bucketKey: string, definition: ChartDefinition): string {
+  if (definition.groupingField) {
+    const [_group, key] = bucketKey.split('::', 2);
+    return key || bucketKey;
+  }
+  return bucketKey;
+}
+
+function getParentDateBucketKey(
+  bucketKey: string,
+  transform: ChartXTransformFunction,
+  isGrouped: boolean
+): string | null {
+  if (isGrouped) {
+    const [group, key] = bucketKey.split('::', 2);
+    if (!key) {
+      return null; // no parent for grouped bucket
+    }
+    return `${group}::${getParentDateBucketKey(key, transform, false)}`;
+  }
+
   switch (transform) {
     case 'date:year':
       return null; // no parent for year
@@ -345,19 +427,30 @@ function createParentChartAggregation(chart: ProcessedChart): ProcessedChart | n
     validYRows: { ...chart.validYRows }, // copy valid Y rows
     topDistinctValues: { ...chart.topDistinctValues }, // copy top distinct values
     availableColumns: chart.availableColumns,
+    groups: [...chart.groups], // copy groups
+    groupSet: new Set(chart.groups), // create a set from the groups
+    bucketKeysSet: new Set<string>(), // initialize empty set for bucket keys
   };
 
-  for (const [bucketKey, bucketValues] of Object.entries(chart.buckets)) {
-    const parentKey = getParentDateBucketKey(bucketKey, chart.definition.xdef.transformFunction);
-    if (!parentKey) {
+  for (const bucketKey of chart.bucketKeysSet) {
+    res.bucketKeysSet.add(getParentDateBucketKey(bucketKey, chart.definition.xdef.transformFunction, false));
+  }
+
+  for (const [groupedBucketKey, bucketValues] of Object.entries(chart.buckets)) {
+    const groupedParentKey = getParentDateBucketKey(
+      groupedBucketKey,
+      chart.definition.xdef.transformFunction,
+      !!chart.definition.groupingField
+    );
+    if (!groupedParentKey) {
       // skip if the bucket is already a parent
       continue;
     }
-    res.bucketKeyDateParsed[parentKey] = getParentKeyParsed(
-      chart.bucketKeyDateParsed[bucketKey],
+    res.bucketKeyDateParsed[extractBucketKeyWithoutGroup(groupedParentKey, chart.definition)] = getParentKeyParsed(
+      chart.bucketKeyDateParsed[extractBucketKeyWithoutGroup(groupedBucketKey, chart.definition)],
       chart.definition.xdef.transformFunction
     );
-    aggregateChartNumericValuesFromChild(res, parentKey, bucketValues);
+    aggregateChartNumericValuesFromChild(res, groupedParentKey, bucketValues);
   }
 
   const bucketKeys = Object.keys(res.buckets).sort();
@@ -400,7 +493,7 @@ export function aggregateChartNumericValuesFromSource(
   row: any
 ) {
   for (const ydef of chart.definition.ydefs) {
-    if (numericColumns[ydef.field] == null) {
+    if (numericColumns[ydef.field] == null && ydef.field != '__count') {
       if (row[ydef.field]) {
         chart.invalidYRows[ydef.field] = (chart.invalidYRows[ydef.field] || 0) + 1; // increment invalid row count if the field is not numeric
       }
@@ -527,16 +620,54 @@ export function fillChartTimelineBuckets(chart: ProcessedChart) {
   const transform = chart.definition.xdef.transformFunction;
 
   let currentParsed = fromParsed;
+  let count = 0;
   while (compareChartDatesParsed(currentParsed, toParsed, transform) <= 0) {
     const bucketKey = stringifyChartDate(currentParsed, transform);
     if (!chart.buckets[bucketKey]) {
       chart.buckets[bucketKey] = {};
+    }
+    if (!chart.bucketKeyDateParsed[bucketKey]) {
       chart.bucketKeyDateParsed[bucketKey] = currentParsed;
     }
+    chart.bucketKeysSet.add(bucketKey);
     currentParsed = incrementChartDate(currentParsed, transform);
+    count++;
+    if (count > ChartLimits.CHART_FILL_LIMIT) {
+      chart.errorMessage = `Too many buckets to fill in chart, limit is ${ChartLimits.CHART_FILL_LIMIT}`;
+      return;
+    }
   }
 }
 
 export function computeChartBucketCardinality(bucket: { [key: string]: any }): number {
-  return _sumBy(Object.keys(bucket), field => bucket[field]);
+  return _sumBy(Object.keys(bucket ?? {}), field => bucket[field]);
+}
+
+export function getChartYRange(chart: ProcessedChart, ydef: ChartYFieldDefinition) {
+  let min = null;
+  let max = null;
+
+  for (const obj of Object.values(chart.buckets)) {
+    const value = obj[ydef.field];
+    if (value != null) {
+      if (min === null || value < min) {
+        min = value;
+      }
+      if (max === null || value > max) {
+        max = value;
+      }
+    }
+  }
+
+  return { min, max };
+}
+
+export function chartsHaveSimilarRange(range1: number, range2: number) {
+  if (range1 < 0 && range2 < 0) {
+    return Math.abs(range1 - range2) / Math.abs(range1) < 0.5;
+  }
+  if (range1 > 0 && range2 > 0) {
+    return Math.abs(range1 - range2) / Math.abs(range1) < 0.5;
+  }
+  return false;
 }

packages/datalib/src/tests/chartProcessor.test.ts

@@ -55,7 +55,7 @@ const DS2 = [
   {
     ts1: '2023-10-03T07:10:00Z',
     ts2: '2024-10-03T07:10:00Z',
-    price1: '13',
+    price1: '22',
     price2: '24',
   },
   {
@@ -116,22 +116,42 @@ describe('Chart processor', () => {
     const processor = new ChartProcessor();
     processor.addRows(...DS1.slice(0, 3));
     processor.finalize();
-    expect(processor.charts.length).toEqual(1);
-    const chart = processor.charts[0];
-    expect(chart.definition.xdef.transformFunction).toEqual('date:day');
-    expect(chart.definition.ydefs).toEqual([
+    // console.log(getChartDebugPrint(processor.charts[0]));
+    expect(processor.charts.length).toEqual(6);
+    const chart1 = processor.charts.find(x => !x.definition.groupingField && x.definition.xdef.field === 'timestamp');
+    expect(chart1.definition.xdef.transformFunction).toEqual('date:day');
+    expect(chart1.definition.ydefs).toEqual([
       expect.objectContaining({
         field: 'value',
       }),
     ]);
-    expect(chart.bucketKeysOrdered).toEqual(['2023-10-01', '2023-10-02', '2023-10-03']);
+    expect(chart1.bucketKeysOrdered).toEqual(['2023-10-01', '2023-10-02', '2023-10-03']);
+
+    const chart2 = processor.charts.find(x => x.definition.groupingField && x.definition.xdef.field === 'timestamp');
+    expect(chart2.definition.xdef.transformFunction).toEqual('date:day');
+    expect(chart2.bucketKeysOrdered).toEqual(['2023-10-01', '2023-10-02', '2023-10-03']);
+    expect(chart2.definition.groupingField).toEqual('category');
+
+    const chart3 = processor.charts.find(x => x.definition.xdef.field === 'category');
+    expect(chart3.bucketKeysOrdered).toEqual(['A', 'B']);
+    expect(chart3.definition.groupingField).toBeUndefined();
+
+    const countCharts = processor.charts.filter(
+      x => x.definition.ydefs.length == 1 && x.definition.ydefs[0].field == '__count'
+    );
+    expect(countCharts.length).toEqual(3);
   });
   test('By month grouped, autedetected', () => {
     const processor = new ChartProcessor();
     processor.addRows(...DS1.slice(0, 4));
     processor.finalize();
-    expect(processor.charts.length).toEqual(1);
-    const chart = processor.charts[0];
+    expect(processor.charts.length).toEqual(6);
+    const chart = processor.charts.find(
+      x =>
+        !x.definition.groupingField &&
+        x.definition.xdef.field === 'timestamp' &&
+        !x.definition.ydefs.find(y => y.field === '__count')
+    );
     expect(chart.definition.xdef.transformFunction).toEqual('date:month');
     expect(chart.bucketKeysOrdered).toEqual([
       '2023-10',
@@ -201,7 +221,7 @@ describe('Chart processor', () => {
     const processor = new ChartProcessor();
     processor.addRows(...DS2);
     processor.finalize();
-    expect(processor.charts.length).toEqual(2);
+    expect(processor.charts.length).toEqual(4);
     expect(processor.charts[0].definition).toEqual(
       expect.objectContaining({
         xdef: expect.objectContaining({
@@ -244,8 +264,8 @@ describe('Chart processor', () => {
     const processor = new ChartProcessor();
     processor.addRows(...DS3);
     processor.finalize();
-    expect(processor.charts.length).toEqual(1);
-    const chart = processor.charts[0];
+    expect(processor.charts.length).toEqual(2);
+    const chart = processor.charts.find(x => !x.definition.ydefs.find(y => y.field === '__count'));
     expect(chart.definition.xdef.transformFunction).toEqual('date:day');
     expect(chart.definition.ydefs).toEqual([
       expect.objectContaining({
@@ -373,4 +393,33 @@ describe('Chart processor', () => {
       expect(chart.buckets).toEqual(expectedBuckets);
     }
   );
+
+  test('Incorrect chart definition', () => {
+    const processor = new ChartProcessor([
+      {
+        chartType: 'bar',
+        xdef: {
+          field: 'category',
+          transformFunction: 'date:day',
+        },
+        ydefs: [],
+      },
+    ]);
+    processor.addRows(...DS1.slice(0, 3));
+    processor.finalize();
+
+    expect(processor.charts.length).toEqual(1);
+    const chart = processor.charts[0];
+    expect(chart.definition.xdef.transformFunction).toEqual('date:day');
+
+    // console.log(getChartDebugPrint(processor.charts[0]));
+
+    // expect(chart.definition.xdef.transformFunction).toEqual('date:day');
+    // expect(chart.definition.ydefs).toEqual([
+    //   expect.objectContaining({
+    //     field: 'value',
+    //   }),
+    // ]);
+    // expect(chart.bucketKeysOrdered).toEqual(['2023-10-01', '2023-10-02', '2023-10-03']);
+  });
 });

packages/filterparser/src/parseFilter.ts

@@ -6,11 +6,13 @@ import { hexStringToArray, parseNumberSafe } from 'dbgate-tools';
 import { FilterBehaviour, TransformType } from 'dbgate-types';
 
 const binaryCondition =
-  (operator, numberDualTesting = false) =>
+  (operator, filterBehaviour: FilterBehaviour = {}) =>
   value => {
+    const { passNumbers, allowNumberDualTesting } = filterBehaviour;
     const numValue = parseNumberSafe(value);
+
     if (
-      numberDualTesting &&
+      allowNumberDualTesting &&
       // @ts-ignore
       !isNaN(numValue)
     ) {
@@ -43,6 +45,21 @@ const binaryCondition =
       };
     }
 
+    // @ts-ignore
+    if (passNumbers && !isNaN(numValue)) {
+      return {
+        conditionType: 'binary',
+        operator,
+        left: {
+          exprType: 'placeholder',
+        },
+        right: {
+          exprType: 'value',
+          value: numValue,
+        },
+      };
+    }
+
     return {
       conditionType: 'binary',
       operator,
@@ -462,18 +479,18 @@ const createParser = (filterBehaviour: FilterBehaviour) => {
     null: () => word('NULL').map(unaryCondition('isNull')),
     isEmpty: r => r.empty.map(unaryCondition('isEmpty')),
     isNotEmpty: r => r.not.then(r.empty).map(unaryCondition('isNotEmpty')),
-    true: () => P.regexp(/true/i).map(binaryFixedValueCondition('1')),
-    false: () => P.regexp(/false/i).map(binaryFixedValueCondition('0')),
+    true: () => P.regexp(/true/i).map(binaryFixedValueCondition(filterBehaviour.passBooleans ? true : '1')),
+    false: () => P.regexp(/false/i).map(binaryFixedValueCondition(filterBehaviour.passBooleans ? false : '0')),
     trueNum: () => word('1').map(binaryFixedValueCondition('1')),
     falseNum: () => word('0').map(binaryFixedValueCondition('0')),
 
-    eq: r => word('=').then(r.value).map(binaryCondition('=', filterBehaviour.allowNumberDualTesting)),
-    ne: r => word('!=').then(r.value).map(binaryCondition('<>', filterBehaviour.allowNumberDualTesting)),
-    ne2: r => word('<>').then(r.value).map(binaryCondition('<>', filterBehaviour.allowNumberDualTesting)),
-    le: r => word('<=').then(r.value).map(binaryCondition('<=', filterBehaviour.allowNumberDualTesting)),
-    ge: r => word('>=').then(r.value).map(binaryCondition('>=', filterBehaviour.allowNumberDualTesting)),
-    lt: r => word('<').then(r.value).map(binaryCondition('<', filterBehaviour.allowNumberDualTesting)),
-    gt: r => word('>').then(r.value).map(binaryCondition('>', filterBehaviour.allowNumberDualTesting)),
+    eq: r => word('=').then(r.value).map(binaryCondition('=', filterBehaviour)),
+    ne: r => word('!=').then(r.value).map(binaryCondition('<>', filterBehaviour)),
+    ne2: r => word('<>').then(r.value).map(binaryCondition('<>', filterBehaviour)),
+    le: r => word('<=').then(r.value).map(binaryCondition('<=', filterBehaviour)),
+    ge: r => word('>=').then(r.value).map(binaryCondition('>=', filterBehaviour)),
+    lt: r => word('<').then(r.value).map(binaryCondition('<', filterBehaviour)),
+    gt: r => word('>').then(r.value).map(binaryCondition('>', filterBehaviour)),
     startsWith: r => word('^').then(r.value).map(likeCondition('like', '#VALUE#%')),
     endsWith: r => word('$').then(r.value).map(likeCondition('like', '%#VALUE#')),
     contains: r => word('+').then(r.value).map(likeCondition('like', '%#VALUE#%')),
@@ -526,8 +543,12 @@ const createParser = (filterBehaviour: FilterBehaviour) => {
     allowedElements.push('exists', 'notExists');
   }
 
-  if (filterBehaviour.supportArrayTesting) {
-    allowedElements.push('emptyArray', 'notEmptyArray');
+  if (filterBehaviour.supportEmptyArrayTesting) {
+    allowedElements.push('emptyArray');
+  }
+
+  if (filterBehaviour.supportNotEmptyArrayTesting) {
+    allowedElements.push('notEmptyArray');
   }
 
   if (filterBehaviour.supportNullTesting) {

packages/tools/src/DatabaseAnalyser.ts

@@ -42,8 +42,7 @@ function areDifferentRowCounts(db1: DatabaseInfo, db2: DatabaseInfo) {
   }
   return false;
 }
-
-export class DatabaseAnalyser {
+export class DatabaseAnalyser<TClient = any> {
   structure: DatabaseInfo;
   modifications: DatabaseModification[];
   singleObjectFilter: any;
@@ -51,7 +50,7 @@ export class DatabaseAnalyser {
   dialect: SqlDialect;
   logger: Logger;
 
-  constructor(public dbhan: DatabaseHandle, public driver: EngineDriver, version) {
+  constructor(public dbhan: DatabaseHandle<TClient>, public driver: EngineDriver, version) {
     this.dialect = (driver?.dialectByVersion && driver?.dialectByVersion(version)) || driver?.dialect;
     this.logger = logger;
   }

packages/tools/src/filterBehaviours.ts

@@ -24,6 +24,7 @@ export const stringFilterBehaviour: FilterBehaviour = {
 export const logicalFilterBehaviour: FilterBehaviour = {
   supportBooleanValues: true,
   supportNullTesting: true,
+  supportBooleanOrNull: true,
   supportSqlCondition: true,
 };
 
@@ -36,7 +37,8 @@ export const datetimeFilterBehaviour: FilterBehaviour = {
 
 export const mongoFilterBehaviour: FilterBehaviour = {
   supportEquals: true,
-  supportArrayTesting: true,
+  supportEmptyArrayTesting: true,
+  supportNotEmptyArrayTesting: true,
   supportNumberLikeComparison: true,
   supportStringInclusion: true,
   supportBooleanValues: true,
@@ -57,11 +59,38 @@ export const evalFilterBehaviour: FilterBehaviour = {
   allowStringToken: true,
 };
 
+export const firestoreFilterBehaviours: FilterBehaviour = {
+  supportEquals: true,
+  supportEmpty: false,
+  supportNumberLikeComparison: true,
+  supportDatetimeComparison: false,
+  supportNullTesting: true,
+  supportBooleanValues: true,
+  supportEmptyArrayTesting: true,
+
+  supportStringInclusion: false,
+  supportDatetimeSymbols: false,
+  supportExistsTesting: false,
+  supportSqlCondition: false,
+
+  allowStringToken: true,
+  allowNumberToken: true,
+  allowHexString: true,
+  allowNumberDualTesting: false,
+  allowObjectIdTesting: false,
+
+  passBooleans: true,
+  passNumbers: true,
+
+  disableOr: true,
+};
+
 export const standardFilterBehaviours: { [id: string]: FilterBehaviour } = {
   numberFilterBehaviour,
   stringFilterBehaviour,
   logicalFilterBehaviour,
   datetimeFilterBehaviour,
   mongoFilterBehaviour,
+  firestoreFilterBehaviours,
   evalFilterBehaviour,
 };

packages/tools/src/stringTools.ts

@@ -75,6 +75,37 @@ export function parseCellValue(value, editorTypes?: DataEditorTypesBehaviour) {
     }
   }
 
+  if (editorTypes?.parseGeopointAsDollar) {
+    const m = value.match(/^([\d\.]+)\s*°\s*([NS]),\s*([\d\.]+)\s*°\s*([EW])$/i);
+    if (m) {
+      let latitude = parseFloat(m[1]);
+      const latDir = m[2].toUpperCase();
+      let longitude = parseFloat(m[3]);
+      const lonDir = m[4].toUpperCase();
+
+      if (latDir === 'S') latitude = -latitude;
+      if (lonDir === 'W') longitude = -longitude;
+
+      return {
+        $geoPoint: {
+          latitude,
+          longitude,
+        },
+      };
+    }
+  }
+
+  if (editorTypes?.parseFsDocumentRefAsDollar) {
+    const trimmedValue = value.replace(/\s/g, '');
+    if (trimmedValue.startsWith('$ref:')) {
+      return {
+        $fsDocumentRef: {
+          documentPath: trimmedValue.slice(5),
+        },
+      };
+    }
+  }
+
   if (editorTypes?.parseJsonNull) {
     if (value == 'null') return null;
   }
@@ -246,6 +277,32 @@ export function stringifyCellValue(
     }
   }
 
+  if (editorTypes?.parseGeopointAsDollar) {
+    if (value?.$geoPoint) {
+      const { latitude, longitude } = value.$geoPoint;
+      if (_isNumber(latitude) && _isNumber(longitude)) {
+        const latAbs = Math.abs(latitude);
+        const lonAbs = Math.abs(longitude);
+        const latDir = latitude >= 0 ? 'N' : 'S';
+        const lonDir = longitude >= 0 ? 'E' : 'W';
+
+        return {
+          value: `${latAbs}° ${latDir}, ${lonAbs}° ${lonDir}`,
+          gridStyle: 'valueCellStyle',
+        };
+      }
+    }
+  }
+
+  if (editorTypes?.parseFsDocumentRefAsDollar) {
+    if (value?.$fsDocumentRef) {
+      return {
+        value: `$ref: ${value.$fsDocumentRef.documentPath ?? ''}`,
+        gridStyle: 'valueCellStyle',
+      };
+    }
+  }
+
   if (_isArray(value)) {
     switch (intent) {
       case 'gridCellIntent':
@@ -644,6 +701,7 @@ export function parseNumberSafe(value) {
 const frontMatterRe = /^--\ >>>[ \t\r]*\n(.*)\n-- <<<[ \t\r]*\n/s;
 
 export function getSqlFrontMatter(text: string, yamlModule) {
+  if (!text || !_isString(text)) return null;
   const match = text.match(frontMatterRe);
   if (!match) return null;
   const yamlContentMapped = match[1].replace(/^--[ ]?/gm, '');
@@ -651,6 +709,7 @@ export function getSqlFrontMatter(text: string, yamlModule) {
 }
 
 export function removeSqlFrontMatter(text: string) {
+  if (!text || !_isString(text)) return null;
   return text.replace(frontMatterRe, '');
 }
 
@@ -673,5 +732,5 @@ export function setSqlFrontMatter(text: string, data: { [key: string]: any }, ya
     .map(line => '-- ' + line)
     .join('\n');
   const frontMatterContent = `-- >>>\n${yamlContentMapped}\n-- <<<\n`;
-  return frontMatterContent + textClean;
+  return frontMatterContent + (textClean || '');
 }

packages/tools/src/structureTools.ts

@@ -5,6 +5,7 @@ import type {
   ViewInfo,
   CollectionInfo,
   NamedObjectInfo,
+  EngineDriver,
 } from 'dbgate-types';
 import _flatten from 'lodash/flatten';
 import _uniq from 'lodash/uniq';
@@ -304,3 +305,11 @@ export function skipDbGateInternalObjects(db: DatabaseInfo) {
     tables: (db.tables || []).filter(tbl => tbl.pureName != 'dbgate_deploy_journal'),
   };
 }
+
+export function adaptDatabaseInfo(db: DatabaseInfo, driver: EngineDriver): DatabaseInfo {
+  const modelAdapted = {
+    ...db,
+    tables: db.tables.map(table => driver.adaptTableInfo(table)),
+  };
+  return modelAdapted;
+}

packages/tools/src/yamlModelConv.ts

@@ -31,6 +31,11 @@ export interface IndexInfoYaml {
   included?: string[];
 }
 
+export interface UniqueInfoYaml {
+  name: string;
+  columns: string[];
+}
+
 export interface TableInfoYaml {
   name: string;
   // schema?: string;
@@ -38,6 +43,7 @@ export interface TableInfoYaml {
   primaryKey?: string[];
   sortingKey?: string[];
   indexes?: IndexInfoYaml[];
+  uniques?: UniqueInfoYaml[];
 
   insertKey?: string[];
   insertOnly?: string[];
@@ -121,6 +127,12 @@ export function tableInfoToYaml(table: TableInfo): TableInfoYaml {
       return idx;
     });
   }
+  if (tableCopy.uniques?.length > 0) {
+    res.uniques = tableCopy.uniques.map(unique => ({
+      name: unique.constraintName,
+      columns: unique.columns.map(x => x.columnName),
+    }));
+  }
   return res;
 }
 
@@ -165,6 +177,12 @@ export function tableInfoFromYaml(table: TableInfoYaml, allTables: TableInfoYaml
         ...(index.included || []).map(columnName => ({ columnName, isIncludedColumn: true })),
       ],
     })),
+    uniques: table.uniques?.map(unique => ({
+      constraintName: unique.name,
+      pureName: table.name,
+      constraintType: 'unique',
+      columns: unique.columns.map(columnName => ({ columnName })),
+    })),
   };
   if (table.primaryKey) {
     res.primaryKey = {

packages/types/dbinfo.d.ts

@@ -108,6 +108,8 @@ export interface CollectionInfo extends DatabaseObjectInfo {
   // unique combination of columns (should be contatenation of partitionKey and clusterKey)
   uniqueKey?: ColumnReference[];
 
+  autoValueColumns?: ColumnReference[];
+
   // partition key columns
   partitionKey?: ColumnReference[];
 

packages/types/dialect.d.ts

@@ -21,6 +21,7 @@ export interface SqlDialect {
   enableForeignKeyChecks?: boolean;
   requireStandaloneSelectForScopeIdentity?: boolean;
   allowMultipleValuesInsert?: boolean;
+  useServerDatabaseFile?: boolean;
 
   dropColumnDependencies?: string[];
   changeColumnDependencies?: string[];

packages/types/engines.d.ts

@@ -23,6 +23,28 @@ export interface StreamOptions {
   info?: (info) => void;
 }
 
+export type CollectionOperationInfo =
+  | {
+      type: 'createCollection';
+      collection: {
+        name: string;
+      };
+    }
+  | {
+      type: 'dropCollection';
+      collection: string;
+    }
+  | {
+      type: 'renameCollection';
+      collection: string;
+      newName: string;
+    }
+  | {
+      type: 'cloneCollection';
+      collection: string;
+      newName: string;
+    };
+
 export interface RunScriptOptions {
   useTransaction: boolean;
   logScriptItems?: boolean;
@@ -120,6 +142,8 @@ export interface DataEditorTypesBehaviour {
   parseHexAsBuffer?: boolean;
   parseObjectIdAsDollar?: boolean;
   parseDateAsDollar?: boolean;
+  parseGeopointAsDollar?: boolean;
+  parseFsDocumentRefAsDollar?: boolean;
 
   explicitDataType?: boolean;
   supportNumberType?: boolean;
@@ -217,7 +241,7 @@ export interface EngineDriver<TClient = any> extends FilterBehaviourProvider {
   defaultSocketPath?: string;
   authTypeLabel?: string;
   importExportArgs?: any[];
-  connect({ server, port, user, password, database }): Promise<DatabaseHandle<TClient>>;
+  connect({ server, port, user, password, database, connectionDefinition }): Promise<DatabaseHandle<TClient>>;
   close(dbhan: DatabaseHandle<TClient>): Promise<any>;
   query(dbhan: DatabaseHandle<TClient>, sql: string, options?: QueryOptions): Promise<QueryResult>;
   stream(dbhan: DatabaseHandle<TClient>, sql: string, options: StreamOptions);
@@ -264,7 +288,7 @@ export interface EngineDriver<TClient = any> extends FilterBehaviourProvider {
   dropDatabase(dbhan: DatabaseHandle<TClient>, name: string): Promise;
   getQuerySplitterOptions(usage: 'stream' | 'script' | 'editor' | 'import'): any;
   script(dbhan: DatabaseHandle<TClient>, sql: string, options?: RunScriptOptions): Promise;
-  operation(dbhan: DatabaseHandle<TClient>, operation: {}, options?: RunScriptOptions): Promise;
+  operation(dbhan: DatabaseHandle<TClient>, operation: CollectionOperationInfo, options?: RunScriptOptions): Promise;
   getNewObjectTemplates(): NewObjectTemplate[];
   // direct call of dbhan.client method, only some methods could be supported, on only some drivers
   callMethod(dbhan: DatabaseHandle<TClient>, method, args);

packages/types/filter-type.d.ts

@@ -9,11 +9,18 @@ export interface FilterBehaviour {
   supportExistsTesting?: boolean;
   supportBooleanValues?: boolean;
   supportSqlCondition?: boolean;
-  supportArrayTesting?: boolean;
+  supportEmptyArrayTesting?: boolean;
+  supportNotEmptyArrayTesting?: boolean;
+  supportBooleanOrNull?: boolean;
 
   allowStringToken?: boolean;
   allowNumberToken?: boolean;
   allowHexString?: boolean;
   allowNumberDualTesting?: boolean;
   allowObjectIdTesting?: boolean;
+
+  passBooleans?: boolean;
+  passNumbers?: boolean;
+
+  disableOr?: boolean;
 }

packages/types/query.d.ts

@@ -15,3 +15,92 @@ export interface QueryResult {
   columns?: QueryResultColumn[];
   rowsAffected?: number;
 }
+
+export type LeftOperand = {
+  exprType: 'placeholder' | 'column';
+  columnName?: string;
+};
+
+export type RightOperand = {
+  exprType: 'value';
+  value: any;
+};
+
+export type BinaryCondition = {
+  conditionType: 'binary';
+  operator: '=' | '!=' | '<>' | '<' | '<=' | '>' | '>=';
+  left: LeftOperand;
+  right: RightOperand;
+};
+
+export type AndCondition = {
+  conditionType: 'and';
+  conditions: FilterCondition[];
+};
+
+export type OrCondition = {
+  conditionType: 'or';
+  conditions: FilterCondition[];
+};
+
+export type NullCondition = {
+  conditionType: 'isNull' | 'isNotNull';
+  expr: LeftOperand;
+};
+
+export type NotCondition = {
+  conditionType: 'not';
+  condition: FilterCondition;
+};
+
+export type LikeCondition = {
+  conditionType: 'like';
+  left: LeftOperand;
+  right: RightOperand;
+};
+
+export type PredicateCondition = {
+  conditionType: 'specificPredicate';
+  predicate: 'exists' | 'notExists' | 'emptyArray' | 'notEmptyArray';
+  expr: LeftOperand;
+};
+
+export type InCondition = {
+  conditionType: 'in';
+  expr: LeftOperand;
+  values: any[];
+};
+
+export type FilterCondition =
+  | BinaryCondition
+  | AndCondition
+  | OrCondition
+  | NullCondition
+  | NotCondition
+  | LikeCondition
+  | PredicateCondition
+  | InCondition;
+
+export type SortItem = {
+  columnName: string;
+  direction?: 'ASC' | 'DESC';
+};
+
+export type AggregateColumn = {
+  aggregateFunction: 'count' | 'sum' | 'avg' | 'min' | 'max';
+  columnArgument?: string;
+  alias: string;
+};
+
+export type CollectionAggregate = {
+  condition?: FilterCondition;
+  groupByColumns: string[];
+  aggregateColumns: AggregateColumn[];
+};
+
+export type FullQueryOptions = {
+  condition?: FilterCondition;
+  sort?: SortItem[];
+  limit?: number;
+  skip?: number;
+};

packages/web/package.json

@@ -64,10 +64,12 @@
     "chartjs-plugin-zoom": "^1.2.0",
     "date-fns": "^4.1.0",
     "debug": "^4.3.4",
+    "flatpickr": "^4.6.13",
     "fuzzy": "^0.1.3",
     "highlight.js": "^11.11.1",
     "interval-operations": "^1.0.7",
     "leaflet": "^1.8.0",
+    "openai": "^5.10.1",
     "wellknown": "^0.5.0",
     "xml-formatter": "^3.6.4"
   }

packages/web/public/global.css

@@ -117,7 +117,7 @@ body {
   max-width: 16.6666%;
 }
 
-.largeFormMarker input[type='text'], .largeFormMarker input[type='number'], .largeFormMarker input[type='password'], .largeFormMarker textarea {
+.largeFormMarker input[type='text'], .largeFormMarker input[type='number'], .largeFormMarker input[type='password'], .largeFormMarker textarea  {
   width: 100%;
   padding: 10px 10px;
   font-size: 14px;
@@ -126,6 +126,13 @@ body {
   border: 1px solid var(--theme-border);
 }
 
+.input1 {
+  padding: 5px 5px;
+  font-size: 14px;
+  box-sizing: border-box;
+  border-radius: 4px;
+  border: 1px solid var(--theme-border);
+}
 
 .largeFormMarker select {
   width: 100%;

packages/web/src/App.svelte

@@ -60,10 +60,9 @@
         installNewCloudTokenListener();
         initializeAppUpdates();
         installCloudListeners();
+        refreshPublicCloudFiles();
       }
 
-      refreshPublicCloudFiles();
-
       loadedApi = loadedApiValue;
 
       if (!loadedApi) {

packages/web/src/EnterLicensePage.svelte

@@ -50,7 +50,7 @@
           on:click={async e => {
             sessionStorage.setItem('continueTrialConfirmed', '1');
             const { licenseKey } = e.detail;
-            const resp = await apiCall('config/save-license-key', { licenseKey });
+            const resp = await apiCall('config/save-license-key', { licenseKey, tryToRenew: true });
             if (resp?.status == 'ok') {
               internalRedirectTo(isOneOfPage('admin-license') ? '/admin.html' : '/index.html');
             } else {

packages/web/src/appobj/ColumnAppObject.svelte

@@ -24,6 +24,7 @@
   import { DEFAULT_OBJECT_SEARCH_SETTINGS, extensions } from '../stores';
   import { filterName, findEngineDriver } from 'dbgate-tools';
   import { useConnectionInfo } from '../utility/metadataLoaders';
+  import { _t } from '../translations';
 
   export let data;
 
@@ -51,12 +52,12 @@
     const menu = [];
 
     if (!driver.dialect.disableNonPrimaryKeyRename || isPrimaryKey) {
-      menu.push({ text: 'Rename column', onClick: handleRenameColumn });
+      menu.push({ text: _t('column.renameColumn', { defaultMessage: 'Rename column' }), onClick: handleRenameColumn });
     }
 
     menu.push(
-      { text: 'Drop column', onClick: handleDropColumn },
-      { text: 'Copy name', onClick: () => navigator.clipboard.writeText(data.columnName) }
+      { text: _t('column.dropColumn', { defaultMessage: 'Drop column' }), onClick: handleDropColumn },
+      { text: _t('column.copyName', { defaultMessage: 'Copy name' }), onClick: () => navigator.clipboard.writeText(data.columnName) }
     );
 
     return menu;

packages/web/src/appobj/DatabaseAppObject.svelte

@@ -19,7 +19,10 @@
       const count = getOpenedTabs().filter(closeCondition).length;
       if (count > 0) {
         showModal(ConfirmModal, {
-          message: `Closing connection will close ${count} opened tabs, continue?`,
+          message: _t('database.closeConfirm', {
+            defaultMessage: 'Closing connection will close {count} opened tabs, continue?',
+            values: { count },
+          }),
           onConfirm: () => disconnectDatabaseConnection(conid, database, false),
         });
         return;
@@ -49,7 +52,7 @@
     const handleNewQuery = () => {
       const tooltip = `${getConnectionLabel(connection)}\n${name}`;
       openNewTab({
-        title: 'Query #',
+        title: _t('database.newQuery', { defaultMessage: 'Query #' }),
         icon: 'img sql-file',
         tooltip,
         tabComponent: 'QueryTab',
@@ -67,7 +70,11 @@
 
     const handleDropDatabase = () => {
       showModal(ConfirmModal, {
-        message: `Really drop database ${name}? All opened sessions with this database will be forcefully closed.`,
+        message: _t('database.dropConfirm', {
+          defaultMessage:
+            'Really drop database {name}? All opened sessions with this database will be forcefully closed.',
+          values: { name },
+        }),
         onConfirm: () =>
           apiCall('server-connections/drop-database', {
             conid: connection._id,
@@ -128,7 +135,7 @@
 
     const handleBackupDatabase = () => {
       openNewTab({
-        title: 'Backup #',
+        title: _t('database.backup', { defaultMessage: 'Backup #' }),
         icon: 'img db-backup',
         tabComponent: 'BackupDatabaseTab',
         props: {
@@ -140,7 +147,7 @@
 
     const handleRestoreDatabase = () => {
       openNewTab({
-        title: 'Restore #',
+        title: _t('database.restore', { defaultMessage: 'Restore #' }),
         icon: 'img db-restore',
         tabComponent: 'RestoreDatabaseTab',
         props: {
@@ -157,7 +164,7 @@
       });
       openNewTab(
         {
-          title: 'Diagram #',
+          title: _t('database.diagram', { defaultMessage: 'Diagram #' }),
           icon: 'img diagram',
           tabComponent: 'DiagramTab',
           props: {
@@ -201,10 +208,22 @@
       // showSnackbarSuccess(`Saved to archive ${resp.archiveFolder}`);
     };
 
+    const handleDatabaseChat = () => {
+      openNewTab({
+        title: 'Chat',
+        icon: 'img ai',
+        tabComponent: 'DatabaseChatTab',
+        props: {
+          conid: connection._id,
+          database: name,
+        },
+      });
+    };
+
     const handleCompareWithCurrentDb = () => {
       openNewTab(
         {
-          title: 'Compare',
+          title: _t('database.compare', { defaultMessage: 'Compare' }),
           icon: 'img compare',
           tabComponent: 'CompareModelTab',
           props: {
@@ -246,14 +265,14 @@
       }
 
       newQuery({
-        title: 'Export #',
+        title: _t('database.export', { defaultMessage: 'Export #' }),
         initialData: data,
       });
     };
 
     const handleQueryDesigner = () => {
       openNewTab({
-        title: 'Query #',
+        title: _t('database.queryDesigner', { defaultMessage: 'Query #' }),
         icon: 'img query-design',
         tabComponent: 'QueryDesignTab',
         focused: true,
@@ -266,7 +285,7 @@
 
     const handleNewPerspective = () => {
       openNewTab({
-        title: 'Perspective #',
+        title: _t('database.perspective', { defaultMessage: 'Perspective #' }),
         icon: 'img perspective',
         tabComponent: 'PerspectiveTab',
         props: {
@@ -278,7 +297,7 @@
 
     const handleDatabaseProfiler = () => {
       openNewTab({
-        title: 'Profiler',
+        title: _t('database.profiler', { defaultMessage: 'Profiler' }),
         icon: 'img profiler',
         tabComponent: 'ProfilerTab',
         props: {
@@ -305,12 +324,16 @@
 
     const handleGenerateDropAllObjectsScript = () => {
       showModal(ConfirmModal, {
-        message: `This will generate script, after executing this script all objects in ${name} will be dropped. Continue?`,
+        message: _t('database.dropAllObjectsConfirm', {
+          defaultMessage:
+            'This will generate script, after executing this script all objects in {name} will be dropped. Continue?',
+          values: { name },
+        }),
 
         onConfirm: () => {
           openNewTab(
             {
-              title: 'Shell #',
+              title: _t('database.shellTitle', { defaultMessage: 'Shell #' }),
               icon: 'img shell',
               tabComponent: 'ShellTab',
             },
@@ -333,7 +356,7 @@ await dbgateApi.dropAllDbObjects(${JSON.stringify(
     const handleGenerateRunScript = () => {
       openNewTab(
         {
-          title: 'Shell #',
+          title: _t('database.shellTitle', { defaultMessage: 'Shell #' }),
           icon: 'img shell',
           tabComponent: 'ShellTab',
         },
@@ -355,7 +378,9 @@ await dbgateApi.executeQuery(${JSON.stringify(
 
     const handleShowDataDeployer = () => {
       showModal(ChooseArchiveFolderModal, {
-        message: 'Choose archive folder for data deployer',
+        message: _t('database.chooseArchiveFolderForDataDeployer', {
+          defaultMessage: 'Choose archive folder for data deployer',
+        }),
         onConfirm: archiveFolder => {
           openNewTab(
             {
@@ -387,57 +412,110 @@ await dbgateApi.executeQuery(${JSON.stringify(
       driver?.databaseEngineTypes?.includes('sql') || driver?.databaseEngineTypes?.includes('document');
 
     return [
-      hasPermission(`dbops/query`) && { onClick: handleNewQuery, text: 'New query', isNewQuery: true },
+      hasPermission(`dbops/query`) && {
+        onClick: handleNewQuery,
+        text: _t('database.newQuery', { defaultMessage: 'New query' }),
+        isNewQuery: true,
+      },
       hasPermission(`dbops/model/edit`) &&
         !connection.isReadOnly &&
-        driver?.databaseEngineTypes?.includes('sql') && { onClick: handleNewTable, text: 'New table' },
+        driver?.databaseEngineTypes?.includes('sql') && {
+          onClick: handleNewTable,
+          text: _t('database.newTable', { defaultMessage: 'New table' }),
+        },
       !connection.isReadOnly &&
         hasPermission(`dbops/model/edit`) &&
         driver?.databaseEngineTypes?.includes('document') && {
           onClick: handleNewCollection,
-          text: `New ${driver?.collectionSingularLabel ?? 'collection/container'}`,
+          text: _t('database.newCollection', {
+            defaultMessage: 'New {collectionLabel}',
+            values: { collectionLabel: driver?.collectionSingularLabel ?? 'collection/container' },
+          }),
         },
       hasPermission(`dbops/query`) &&
         driver?.databaseEngineTypes?.includes('sql') &&
-        isProApp() && { onClick: handleQueryDesigner, text: 'Design query' },
+        isProApp() && {
+          onClick: handleQueryDesigner,
+          text: _t('database.designQuery', { defaultMessage: 'Design query' }),
+        },
       driver?.databaseEngineTypes?.includes('sql') &&
         isProApp() && {
           onClick: handleNewPerspective,
-          text: 'Design perspective query',
+          text: _t('database.designPerspectiveQuery', { defaultMessage: 'Design perspective query' }),
         },
-      connection.useSeparateSchemas && { onClick: handleRefreshSchemas, text: 'Refresh schemas' },
+      connection.useSeparateSchemas && {
+        onClick: handleRefreshSchemas,
+        text: _t('database.refreshSchemas', { defaultMessage: 'Refresh schemas' }),
+      },
 
       { divider: true },
       isSqlOrDoc &&
         !connection.isReadOnly &&
-        hasPermission(`dbops/import`) && { onClick: handleImport, text: 'Import' },
-      isSqlOrDoc && hasPermission(`dbops/export`) && { onClick: handleExport, text: 'Export' },
+        hasPermission(`dbops/import`) && {
+          onClick: handleImport,
+          text: _t('database.import', { defaultMessage: 'Import' }),
+        },
+      isSqlOrDoc &&
+        hasPermission(`dbops/export`) && {
+          onClick: handleExport,
+          text: _t('database.export', { defaultMessage: 'Export' }),
+        },
       driver?.supportsDatabaseRestore &&
         isProApp() &&
         hasPermission(`dbops/sql-dump/import`) &&
-        !connection.isReadOnly && { onClick: handleRestoreDatabase, text: 'Restore database backup' },
+        !connection.isReadOnly && {
+          onClick: handleRestoreDatabase,
+          text: _t('database.restoreDatabaseBackup', { defaultMessage: 'Restore database backup' }),
+        },
       driver?.supportsDatabaseBackup &&
         isProApp() &&
-        hasPermission(`dbops/sql-dump/export`) && { onClick: handleBackupDatabase, text: 'Create database backup' },
+        hasPermission(`dbops/sql-dump/export`) && {
+          onClick: handleBackupDatabase,
+          text: _t('database.createDatabaseBackup', { defaultMessage: 'Create database backup' }),
+        },
       isSqlOrDoc &&
         !connection.isReadOnly &&
         !connection.singleDatabase &&
         isSqlOrDoc &&
-        hasPermission(`dbops/dropdb`) && { onClick: handleDropDatabase, text: 'Drop database' },
+        hasPermission(`dbops/dropdb`) && {
+          onClick: handleDropDatabase,
+          text: _t('database.dropDatabase', { defaultMessage: 'Drop database' }),
+        },
       { divider: true },
-      driver?.databaseEngineTypes?.includes('sql') && { onClick: handleCopyName, text: 'Copy database name' },
-      driver?.databaseEngineTypes?.includes('sql') && { onClick: handleShowDiagram, text: 'Show diagram' },
+      driver?.databaseEngineTypes?.includes('sql') && {
+        onClick: handleCopyName,
+        text: _t('database.copyDatabaseName', { defaultMessage: 'Copy database name' }),
+      },
+      driver?.databaseEngineTypes?.includes('sql') && {
+        onClick: handleShowDiagram,
+        text: _t('database.showDiagram', { defaultMessage: 'Show diagram' }),
+      },
       driver?.databaseEngineTypes?.includes('sql') &&
-        hasPermission(`dbops/sql-generator`) && { onClick: handleSqlGenerator, text: 'SQL Generator' },
+        hasPermission(`dbops/sql-generator`) && {
+          onClick: handleSqlGenerator,
+          text: _t('database.sqlGenerator', { defaultMessage: 'SQL Generator' }),
+        },
       driver?.supportsDatabaseProfiler &&
         isProApp() &&
-        hasPermission(`dbops/profiler`) && { onClick: handleDatabaseProfiler, text: 'Database profiler' },
+        hasPermission(`dbops/profiler`) && {
+          onClick: handleDatabaseProfiler,
+          text: _t('database.databaseProfiler', { defaultMessage: 'Database profiler' }),
+        },
       // isSqlOrDoc &&
       //   isSqlOrDoc &&
       //   hasPermission(`dbops/model/view`) && { onClick: handleOpenJsonModel, text: 'Open model as JSON' },
       isSqlOrDoc &&
         isProApp() &&
-        hasPermission(`dbops/model/view`) && { onClick: handleExportModel, text: 'Export DB model' },
+        hasPermission(`dbops/model/view`) && {
+          onClick: handleExportModel,
+          text: _t('database.exportDbModel', { defaultMessage: 'Export DB model' }),
+        },
+      isProApp() &&
+        driver?.databaseEngineTypes?.includes('sql') &&
+        hasPermission('dbops/chat') && {
+          onClick: handleDatabaseChat,
+          text: _t('database.databaseChat', { defaultMessage: 'Database chat' }),
+        },
       isSqlOrDoc &&
         _.get($currentDatabase, 'connection._id') &&
         hasPermission('dbops/model/compare') &&
@@ -446,32 +524,41 @@ await dbgateApi.executeQuery(${JSON.stringify(
           (_.get($currentDatabase, 'connection._id') == _.get(connection, '_id') &&
             _.get($currentDatabase, 'name') != _.get(connection, 'name'))) && {
           onClick: handleCompareWithCurrentDb,
-          text: `Compare with ${_.get($currentDatabase, 'name')}`,
+          text: _t('database.compareWithCurrentDb', {
+            defaultMessage: 'Compare with {name}',
+            values: { name: _.get($currentDatabase, 'name') },
+          }),
         },
 
-      driver?.databaseEngineTypes?.includes('keyvalue') && { onClick: handleGenerateScript, text: 'Generate script' },
+      driver?.databaseEngineTypes?.includes('keyvalue') && {
+        onClick: handleGenerateScript,
+        text: _t('database.generateScript', { defaultMessage: 'Generate script' }),
+      },
 
       ($openedSingleDatabaseConnections.includes(connection._id) ||
         (_.get($currentDatabase, 'connection._id') == _.get(connection, '_id') &&
-          _.get($currentDatabase, 'name') == name)) && { onClick: handleDisconnect, text: 'Disconnect' },
+          _.get($currentDatabase, 'name') == name)) && {
+        onClick: handleDisconnect,
+        text: _t('database.disconnect', { defaultMessage: 'Disconnect' }),
+      },
 
       { divider: true },
 
       driver?.databaseEngineTypes?.includes('sql') &&
         hasPermission(`dbops/dropdb`) && {
           onClick: handleGenerateDropAllObjectsScript,
-          text: 'Shell: Drop all objects',
+          text: _t('database.shellDropAllObjects', { defaultMessage: 'Shell: Drop all objects' }),
         },
 
       {
         onClick: handleGenerateRunScript,
-        text: 'Shell: Run script',
+        text: _t('database.shellRunScript', { defaultMessage: 'Shell: Run script' }),
       },
 
       driver?.databaseEngineTypes?.includes('sql') &&
         hasPermission(`dbops/import`) && {
           onClick: handleShowDataDeployer,
-          text: 'Data deployer',
+          text: _t('database.dataDeployer', { defaultMessage: 'Data deployer' }),
         },
 
       { divider: true },
@@ -548,6 +635,7 @@ await dbgateApi.executeQuery(${JSON.stringify(
   import { extractShellConnection } from '../impexp/createImpExpScript';
   import { getNumberIcon } from '../icons/FontIcon.svelte';
   import { getDatabaseClickActionSetting } from '../settings/settingsTools';
+  import { _t } from '../translations';
 
   export let data;
   export let passProps;

packages/web/src/appobj/SavedFileAppObject.svelte

@@ -1,5 +1,6 @@
 <script lang="ts" context="module">
-  import { filterName, getConnectionLabel } from 'dbgate-tools';
+  import { filterName, getConnectionLabel, getSqlFrontMatter } from 'dbgate-tools';
+  import yaml from 'js-yaml';
 
   interface FileTypeHandler {
     icon: string;
@@ -9,6 +10,7 @@
     currentConnection: boolean;
     extension: string;
     label: string;
+    switchDatabaseOnOpen?: (data: any) => Promise<boolean>;
   }
 
   const sql: FileTypeHandler = {
@@ -19,6 +21,21 @@
     currentConnection: true,
     extension: 'sql',
     label: 'SQL file',
+    switchDatabaseOnOpen: async data => {
+      const frontMatter = getSqlFrontMatter(data, yaml);
+      if (frontMatter?.connectionId) {
+        const connection = await getConnectionInfo({ conid: frontMatter.connectionId });
+        // console.log('Switching database to', frontMatter.databaseName, 'on connection', connection);
+        if (connection && frontMatter.databaseName) {
+          currentDatabase.set({
+            connection,
+            name: frontMatter.databaseName,
+          });
+          return true;
+        }
+      }
+      return false;
+    },
   };
 
   const shell: FileTypeHandler = {
@@ -161,6 +178,7 @@
   import AppObjectCore from './AppObjectCore.svelte';
   import { isProApp } from '../utility/proTools';
   import { saveFileToDisk } from '../utility/exportFileTools';
+  import { getConnectionInfo } from '../utility/metadataLoaders';
 
   export let data;
 
@@ -281,6 +299,10 @@
     let tooltip = undefined;
     const connProps: any = {};
 
+    if (handler.switchDatabaseOnOpen) {
+      await handler.switchDatabaseOnOpen(dataContent);
+    }
+
     if (handler.currentConnection) {
       const connection = _.get($currentDatabase, 'connection') || {};
       const database = _.get($currentDatabase, 'name');

packages/web/src/buttons/DropDownButton.svelte

@@ -11,6 +11,7 @@
   export let narrow = false;
   export let square = true;
   export let disabled = false;
+  export let title = undefined;
 
   let domButton;
   let isLoading = false;
@@ -40,6 +41,7 @@
   bind:this={domButton}
   {disabled}
   data-testid={$$props['data-testid']}
+  {title}
 >
   <FontIcon icon={isLoading ? 'icon loading' : icon} />
 </InlineButton>

packages/web/src/buttons/FormStyledDropDownButton.svelte

@@ -0,0 +1,78 @@
+<script lang="ts">
+  import { currentDropDownMenu } from '../stores';
+  import FontIcon from '../icons/FontIcon.svelte';
+
+  export let value;
+  export let menu = [];
+  export let asyncMenu = undefined;
+  export let disabled = false;
+  export let outline = false;
+  export let skipWidth = false;
+  export let icon = 'icon chevron-down';
+
+  let domButton;
+  let isLoading = false;
+
+  async function handleClick() {
+    if (disabled) return;
+
+    let items = menu;
+
+    if (asyncMenu) {
+      isLoading = true;
+      items = await asyncMenu();
+      isLoading = false;
+    }
+
+    const rect = domButton.getBoundingClientRect();
+    const left = rect.left;
+    const top = rect.bottom;
+    currentDropDownMenu.set({ left, top, items });
+  }
+</script>
+
+<div on:click={handleClick} class:disabled class:outline class:skipWidth bind:this={domButton}>
+  {value}
+  <FontIcon icon={isLoading ? 'icon loading' : icon} padLeft />
+</div>
+
+<style>
+  div {
+    border: 1px solid var(--theme-bg-button-inv-2);
+    padding: 2px;
+    padding-bottom: 4px;
+    margin: 2px;
+    background-color: var(--theme-bg-button-inv);
+    color: var(--theme-font-inv-1);
+    border-radius: 2px;
+    display: inline-block;
+    cursor: pointer;
+  }
+
+  div:not(.skipWidth) {
+    width: 100px;
+  }
+
+  div:hover:not(.disabled):not(.outline) {
+    background-color: var(--theme-bg-button-inv-2);
+  }
+  div:active:not(.disabled):not(.outline) {
+    background-color: var(--theme-bg-button-inv-3);
+  }
+  div.disabled {
+    background-color: var(--theme-bg-button-inv-3);
+    color: var(--theme-font-inv-3);
+  }
+
+  div.outline {
+    background-color: transparent;
+    color: var(--theme-font-2);
+    border: 1px solid var(--theme-bg-button-inv-2);
+  }
+
+  input.outline:hover:not(.disabled) {
+    color: var(--theme-bg-button-inv-3);
+    border: 2px solid var(--theme-bg-button-inv-3);
+    margin: 1px;
+  }
+</style>

packages/web/src/buttons/NewObjectButton.svelte

@@ -0,0 +1,86 @@
+<script lang="ts">
+  import FontIcon from '../icons/FontIcon.svelte';
+  import { isProApp } from '../utility/proTools';
+
+  export let icon;
+  export let title;
+  export let description;
+  export let enabled;
+  export let colorClass;
+  export let disabledMessage = undefined;
+  export let isProFeature;
+
+  $: disabled = !enabled;
+</script>
+
+<div
+  class="new-object-button"
+  on:click
+  class:enabled
+  class:disabled
+  data-testid={$$props['data-testid']}
+  title={disabled
+    ? isProFeature && !isProApp()
+      ? 'This feature is available only in DbGate Premium'
+      : disabledMessage
+    : undefined}
+>
+  <div class="icon">
+    <FontIcon {icon} colorClass={enabled ? colorClass : null} />
+  </div>
+  <span class="title">{title}</span>
+  {#if description}
+    <div class="description">{description}</div>
+  {/if}
+</div>
+
+<style>
+  .new-object-button {
+    width: 150px;
+    height: 150px;
+    background-color: var(--theme-bg-1);
+    border-radius: 4px;
+    border: 1px solid var(--theme-border);
+
+    display: flex;
+    flex-direction: column;
+    align-items: center;
+  }
+
+  .new-object-button.enabled {
+    cursor: pointer;
+  }
+  .new-object-button.enabled:hover {
+    background-color: var(--theme-bg-2);
+  }
+  .icon {
+    font-size: 3em;
+    margin-top: 20px;
+    color: var(--theme-font-1);
+  }
+  .title {
+    margin-top: 0.2em;
+    font-weight: bold;
+  }
+  .description {
+    margin-top: 0.2em;
+    margin-left: 0.5em;
+    margin-right: 0.5em;
+    font-size: 0.9em;
+    color: var(--theme-font-2);
+  }
+
+  .new-object-button.disabled .title {
+    color: var(--theme-font-2);
+  }
+  .new-object-button.disabled .description {
+    color: var(--theme-font-2);
+  }
+  .new-object-button.disabled .icon {
+    color: var(--theme-font-2);
+  }
+  .new-object-button.disabled {
+    cursor: not-allowed;
+    opacity: 0.5;
+  }
+</style>

packages/web/src/celldata/XmlCellView.svelte

@@ -1,5 +1,5 @@
 <script lang="ts">
-  import XmlHighlighter from './XmlHighlighter.svelte';
+  import XmlHighlighter from '../elements/XmlHighlighter.svelte';
 
   export let selection;
 </script>

packages/web/src/clientAuth.ts

@@ -3,6 +3,7 @@ import { getConfig } from './utility/metadataLoaders';
 import { isAdminPage } from './utility/pageDefs';
 import getElectron from './utility/getElectron';
 import { isProApp } from './utility/proTools';
+import { cloudSigninTokenHolder, selectedWidget } from './stores';
 
 export function isOauthCallback() {
   const params = new URLSearchParams(location.search);
@@ -114,6 +115,12 @@ export function handleOauthCallback() {
     return true;
   }
 
+  const cloudSid = params.get('dbgate-cloud-sid');
+  if (cloudSid) {
+    sessionStorage.setItem('dbgate-cloud-sid', cloudSid);
+    internalRedirectTo(`/`);
+  }
+
   return false;
 }
 
@@ -187,6 +194,18 @@ export async function handleAuthOnStartup(config) {
     }
   }
 
+  async function checkDbGateCloudLogin() {
+    const sid = sessionStorage.getItem('dbgate-cloud-sid');
+    if (sid) {
+      const tokenHolder = await apiCall('auth/cloud-login-redirected', { sid });
+      if (tokenHolder) {
+        sessionStorage.removeItem('dbgate-cloud-sid');
+        cloudSigninTokenHolder.set(tokenHolder);
+        selectedWidget.set('cloud-private');
+      }
+    }
+  }
+
   if (page == 'error') return;
   if (checkConfigError()) return;
 
@@ -199,6 +218,7 @@ export async function handleAuthOnStartup(config) {
   if (page == 'license' || page == 'admin-license') return;
   if (checkTrialDaysLeft()) return;
   if (checkInvalidLicense()) return;
+  checkDbGateCloudLogin();
 
   // if (config.configurationError) {
   //   internalRedirectTo(`/error.html`);
@@ -291,9 +311,11 @@ export async function doLogout() {
   const category = getAuthCategory(config);
 
   if (category == 'admin') {
+    await apiCall('auth/logout-admin');
     localStorage.removeItem('adminAccessToken');
     internalRedirectTo('/admin-login.html?is-admin=true');
   } else if (category == 'token') {
+    await apiCall('auth/logout-user');
     localStorage.removeItem('accessToken');
     if (config.logoutUrl) {
       window.location.href = config.logoutUrl;

packages/web/src/commands/stdCommands.ts

@@ -5,6 +5,7 @@ import {
   emptyConnectionGroupNames,
   extensions,
   getAppUpdaterActive,
+  getCloudSigninTokenHolder,
   getExtensions,
   getVisibleToolbar,
   visibleToolbar,
@@ -128,11 +129,12 @@ registerCommand({
   id: 'new.connectionOnCloud',
   toolbar: true,
   icon: 'img cloud-connection',
-  toolbarName: 'Add connection on cloud',
+  toolbarName: 'Add connection',
   category: 'New',
   toolbarOrder: 1,
   name: 'Connection on Cloud',
-  testEnabled: () => !getCurrentConfig()?.runAsPortal && !getCurrentConfig()?.storageDatabase,
+  testEnabled: () =>
+    !getCurrentConfig()?.runAsPortal && !getCurrentConfig()?.storageDatabase && !!getCloudSigninTokenHolder(),
   onClick: () => {
     openNewTab({
       title: 'New Connection on Cloud',
@@ -652,7 +654,10 @@ registerCommand({
   name: 'SQL Generator',
   toolbar: true,
   icon: 'icon sql-generator',
-  testEnabled: () => getCurrentDatabase() != null && hasPermission(`dbops/sql-generator`),
+  testEnabled: () =>
+    getCurrentDatabase() != null &&
+    hasPermission(`dbops/sql-generator`) &&
+    findEngineDriver(getCurrentDatabase()?.connection, getExtensions())?.databaseEngineTypes?.includes('sql'),
   onClick: () =>
     showModal(SqlGeneratorModal, {
       conid: getCurrentDatabase()?.connection?._id,
@@ -660,6 +665,80 @@ registerCommand({
     }),
 });
 
+registerCommand({
+  id: 'database.export',
+  category: 'Database',
+  name: 'Export database',
+  toolbar: true,
+  icon: 'icon export',
+  testEnabled: () => getCurrentDatabase() != null,
+  onClick: () => {
+    openImportExportTab({
+      targetStorageType: getDefaultFileFormat(getExtensions()).storageType,
+      sourceStorageType: 'database',
+      sourceConnectionId: getCurrentDatabase()?.connection?._id,
+      sourceDatabaseName: getCurrentDatabase()?.name,
+    });
+  },
+});
+
+if (isProApp()) {
+  registerCommand({
+    id: 'database.compare',
+    category: 'Database',
+    name: 'Compare databases',
+    toolbar: true,
+    icon: 'icon compare',
+    testEnabled: () =>
+      getCurrentDatabase() != null &&
+      findEngineDriver(getCurrentDatabase()?.connection, getExtensions())?.databaseEngineTypes?.includes('sql'),
+    onClick: () => {
+      openNewTab(
+        {
+          title: 'Compare',
+          icon: 'img compare',
+          tabComponent: 'CompareModelTab',
+          props: {
+            conid: getCurrentDatabase()?.connection?._id,
+            database: getCurrentDatabase()?.name,
+          },
+        },
+        {
+          editor: {
+            sourceConid: getCurrentDatabase()?.connection?._id,
+            sourceDatabase: getCurrentDatabase()?.name,
+            targetConid: getCurrentDatabase()?.connection?._id,
+            targetDatabase: getCurrentDatabase()?.name,
+          },
+        }
+      );
+    },
+  });
+
+  registerCommand({
+    id: 'database.chat',
+    category: 'Database',
+    name: 'Database chat',
+    toolbar: true,
+    icon: 'icon ai',
+    testEnabled: () =>
+      getCurrentDatabase() != null &&
+      findEngineDriver(getCurrentDatabase()?.connection, getExtensions())?.databaseEngineTypes?.includes('sql') &&
+      hasPermission('dbops/chat'),
+    onClick: () => {
+      openNewTab({
+        title: 'Chat',
+        icon: 'img ai',
+        tabComponent: 'DatabaseChatTab',
+        props: {
+          conid: getCurrentDatabase()?.connection?._id,
+          database: getCurrentDatabase()?.name,
+        },
+      });
+    },
+  });
+}
+
 if (hasPermission('settings/change')) {
   registerCommand({
     id: 'settings.commands',

packages/web/src/datagrid/CollectionDataGridCore.svelte

@@ -86,6 +86,7 @@
         condition: buildConditionForGrid(props),
         sort: buildSortForGrid(props),
       },
+      auditLogSessionGroup: 'data-grid',
     });
 
     if (response.errorMessage) return response;
@@ -121,7 +122,11 @@
   import _ from 'lodash';
   import { registerQuickExportHandler } from '../buttons/ToolStripExportButton.svelte';
   import registerCommand from '../commands/registerCommand';
-  import { extractShellConnection, extractShellConnectionHostable, extractShellHostConnection } from '../impexp/createImpExpScript';
+  import {
+    extractShellConnection,
+    extractShellConnectionHostable,
+    extractShellHostConnection,
+  } from '../impexp/createImpExpScript';
   import { apiCall } from '../utility/api';
 
   import { registerMenu } from '../utility/contextMenu';

packages/web/src/datagrid/DataFilterControl.svelte

@@ -80,11 +80,12 @@
       );
     }
 
-    if (filterBehaviour.supportArrayTesting) {
-      res.push(
-        { onClick: () => setFilter('NOT EMPTY ARRAY'), text: 'Array is not empty' },
-        { onClick: () => setFilter('EMPTY ARRAY'), text: 'Array is empty' }
-      );
+    if (filterBehaviour.supportNotEmptyArrayTesting) {
+      res.push({ onClick: () => setFilter('NOT EMPTY ARRAY'), text: 'Array is not empty' });
+    }
+
+    if (filterBehaviour.supportEmptyArrayTesting) {
+      res.push({ onClick: () => setFilter('EMPTY ARRAY'), text: 'Array is empty' });
     }
 
     if (filterBehaviour.supportNullTesting) {
@@ -132,7 +133,7 @@
       );
     }
 
-    if (filterBehaviour.supportBooleanValues && filterBehaviour.supportNullTesting) {
+    if (filterBehaviour.supportBooleanOrNull) {
       res.push(
         { onClick: () => setFilter('TRUE, NULL'), text: 'Is True or NULL' },
         { onClick: () => setFilter('FALSE, NULL'), text: 'Is False or NULL' }

packages/web/src/datagrid/DataGridCore.svelte

@@ -424,6 +424,7 @@
   import { _t } from '../translations';
   import { isProApp } from '../utility/proTools';
   import SaveArchiveModal from '../modals/SaveArchiveModal.svelte';
+  import hasPermission from '../utility/hasPermission';
 
   export let onLoadNextData = undefined;
   export let grider = undefined;
@@ -1848,6 +1849,7 @@
     // },
     isProApp() && { command: 'dataGrid.sendToDataDeploy' },
     isProApp() &&
+      hasPermission('dbops/charts') &&
       onOpenChart && {
         text: 'Open chart',
         onClick: () => onOpenChart(),

packages/web/src/datagrid/JslDataGrid.svelte

@@ -8,6 +8,7 @@
 
   import DataGrid from './DataGrid.svelte';
   import JslDataGridCore from './JslDataGridCore.svelte';
+  import { useSettings } from '../utility/metadataLoaders';
 
   export let jslid;
   export let supportsReload = false;
@@ -30,6 +31,7 @@
   // $: columns = ($info && $info.columns) || [];
   const config = writable(createGridConfig());
   const cache = writable(createGridCache());
+  const settingsValue = useSettings();
 
   function handleInitializeFile() {
     infoCounter += 1;
@@ -71,7 +73,8 @@
     infoUsed?.__isDynamicStructure,
     supportsReload,
     !!changeSetState,
-    driver
+    driver,
+    $settingsValue
   );
 
   function handleSetLoadedRows(rows) {

packages/web/src/datagrid/SqlDataGridCore.svelte

@@ -18,41 +18,6 @@
     testEnabled: () => getCurrentEditor() != null && hasPermission('dbops/export'),
     onClick: () => getCurrentEditor().exportGrid(),
   });
-
-  async function loadDataPage(props, offset, limit) {
-    const { display, conid, database } = props;
-
-    const select = display.getPageQuery(offset, limit);
-
-    const response = await apiCall('database-connections/sql-select', {
-      conid,
-      database,
-      select,
-    });
-
-    if (response.errorMessage) return response;
-    return response.rows;
-  }
-
-  function dataPageAvailable(props) {
-    const { display } = props;
-    const select = display.getPageQuery(0, 1);
-    return !!select;
-  }
-
-  async function loadRowCount(props) {
-    const { display, conid, database } = props;
-
-    const select = display.getCountQuery();
-
-    const response = await apiCall('database-connections/sql-select', {
-      conid,
-      database,
-      select,
-    });
-
-    return parseInt(response.rows[0].count);
-  }
 </script>
 
 <script lang="ts">
@@ -217,6 +182,42 @@
   function handleSetLoadedRows(rows) {
     loadedRows = rows;
   }
+
+  async function loadDataPage(props, offset, limit) {
+    const { display, conid, database } = props;
+
+    const select = display.getPageQuery(offset, limit);
+
+    const response = await apiCall('database-connections/sql-select', {
+      conid,
+      database,
+      select,
+      auditLogSessionGroup: 'data-grid',
+    });
+
+    if (response.errorMessage) return response;
+    return response.rows;
+  }
+
+  function dataPageAvailable(props) {
+    const { display } = props;
+    const select = display.getPageQuery(0, 1);
+    return !!select;
+  }
+
+  async function loadRowCount(props) {
+    const { display, conid, database } = props;
+
+    const select = display.getCountQuery();
+
+    const response = await apiCall('database-connections/sql-select', {
+      conid,
+      database,
+      select,
+    });
+
+    return parseInt(response.rows[0].count);
+  }
 </script>
 
 <LoadingDataGridCore

packages/web/src/designer/Designer.svelte

@@ -24,6 +24,18 @@
     onClick: () => getCurrentEditor().exportDiagram(),
     testEnabled: () => getCurrentEditor()?.canExport(),
   });
+
+  registerCommand({
+    id: 'diagram.deleteSelectedTables',
+    category: 'Designer',
+    toolbarName: 'Remove',
+    name: 'Remove selected tables',
+    icon: 'icon delete',
+    toolbar: true,
+    isRelatedToTab: true,
+    onClick: () => getCurrentEditor().deleteSelectedTables(),
+    testEnabled: () => getCurrentEditor()?.areTablesSelected(),
+  });
 </script>
 
 <script lang="ts">
@@ -54,6 +66,7 @@
   import createRef from '../utility/createRef';
   import { isProApp } from '../utility/proTools';
   import dragScroll from '../utility/dragScroll';
+  import FormStyledButton from '../buttons/FormStyledButton.svelte';
 
   export let value;
   export let onChange;
@@ -63,6 +76,7 @@
   export let settings;
   export let referenceComponent;
   export let onReportCounts = undefined;
+  export let allowAddTablesButton = false;
 
   export const activator = createActivator('Designer', true);
 
@@ -423,6 +437,19 @@
     arrange(true, false, rect ? { x: (rect.left + rect.right) / 2, y: (rect.top + rect.bottom) / 2 } : null);
   };
 
+  const handleAddAllTables = async () => {
+    const db = dbInfoExtended;
+    if (!db) return;
+    callChange(current => ({
+      tables: db.tables.map(table => ({
+        ...table,
+        designerId: `${table.pureName}-${uuidv1()}`,
+      })),
+      references: [],
+      autoLayout: true,
+    }));
+  };
+
   const handleChangeTableColor = table => {
     showModal(ChooseColorModal, {
       onChange: color => {
@@ -764,7 +791,11 @@
   }
 
   function getWatermarkHtml() {
-    const replaceLinks = text => text.replace(/\[([^\]]+)\]\(([^)]+)\)/g, '<a href="$2" style="color: var(--theme-font-link)" target="_blank">$1</a>');
+    const replaceLinks = text =>
+      text.replace(
+        /\[([^\]]+)\]\(([^)]+)\)/g,
+        '<a href="$2" style="color: var(--theme-font-link)" target="_blank">$1</a>'
+      );
 
     if (value?.style?.omitExportWatermark) return null;
     if (value?.style?.exportWatermark) {
@@ -890,8 +921,8 @@
   function handleWheel(event) {
     if (event.ctrlKey) {
       event.preventDefault();
-      const zoomIndex = DIAGRAM_ZOOMS.findIndex(x => x == value?.style?.zoomKoef);
-      if (zoomIndex < 0) DIAGRAM_ZOOMS.findIndex(x => x == 1);
+      let zoomIndex = DIAGRAM_ZOOMS.findIndex(x => x == value?.style?.zoomKoef);
+      if (zoomIndex < 0) zoomIndex = DIAGRAM_ZOOMS.findIndex(x => x == 1);
 
       let newZoomIndex = zoomIndex;
       if (event.deltaY < 0) {
@@ -962,6 +993,18 @@
       filtered: _.compact(tables || []).length,
     });
   }
+
+  export function areTablesSelected() {
+    return tables.some(x => x.isSelectedTable);
+  }
+
+  export function deleteSelectedTables() {
+    callChange(current => ({
+      ...current,
+      tables: (current.tables || []).filter(x => !x.isSelectedTable),
+    }));
+    updateFromDbInfo();
+  }
 </script>
 
 <div
@@ -973,6 +1016,12 @@
 >
   {#if !(tables?.length > 0)}
     <div class="empty">Drag &amp; drop tables or views from left panel here</div>
+
+    {#if allowAddTablesButton}
+      <div class="addAllTables">
+        <FormStyledButton value="Add all tables" on:click={handleAddAllTables} />
+      </div>
+    {/if}
   {/if}
 
   <div
@@ -1087,6 +1136,14 @@
     font-size: 20px;
     position: absolute;
   }
+
+  .addAllTables {
+    margin: 50px;
+    margin-top: 100px;
+    font-size: 20px;
+    position: absolute;
+    z-index: 100;
+  }
   .canvas {
     position: relative;
   }