Compare commits

...

23 Commits

Author SHA1 Message Date
Jan Prochazka
0ff4f0d7e9 profile refactoring, fixes 2022-12-18 17:03:47 +01:00
Jan Prochazka
3bbdc56309 max duration profiler measure 2022-12-18 16:18:56 +01:00
Jan Prochazka
2e37788471 profiler charts 2022-12-18 13:48:24 +01:00
Jan Prochazka
9a2631dc09 profiler charts 2022-12-18 12:29:21 +01:00
Jan Prochazka
dbfdaafb86 jsonl filtering fixes 2022-12-18 09:08:03 +01:00
Jan Prochazka
cf3df9cda3 short json value shown in grid 2022-12-17 20:22:42 +01:00
Jan Prochazka
274fcd339b archive file - open in profiler 2022-12-17 20:07:26 +01:00
Jan Prochazka
123e00ecbc mongo profiler formatter 2022-12-17 12:34:28 +01:00
Jan Prochazka
34a4f9adbf save profiler output to archive 2022-12-17 08:57:16 +01:00
Jan Prochazka
0e819bcc45 mongodb profiler 2022-12-16 14:52:49 +01:00
Jan Prochazka
c1ba758b01 mongo profile view - shows collection tab 2022-12-16 09:42:38 +01:00
Jan Prochazka
11daa56335 mongo filter: empty array, not empty array 2022-12-16 08:06:37 +01:00
Jan Prochazka
a9257cf4f8 camel case search 2022-12-15 20:37:38 +01:00
Jan Prochazka
1a2acd764d improved editor margin #422 2022-12-15 18:50:23 +01:00
Jan Prochazka
27b0af6408 Merge branch 'master' of github.com:dbgate/dbgate 2022-12-15 17:39:24 +01:00
Jan Prochazka
3c63738809 fixed missing versioned tables #433 2022-12-15 17:38:02 +01:00
Jan Prochazka
9305e767cd Merge pull request #437 from horaciod/patch-1
Typo in export action
2022-12-15 17:30:24 +01:00
Jan Prochazka
2fddf32e54 fixed broken F5 2022-12-15 17:15:19 +01:00
Jan Prochazka
469fd76f89 upgrade dbgate-query-splitter 2022-12-15 17:10:01 +01:00
Horacio Degiorgi
1f682d91c9 Typo in export action 2022-12-14 10:51:31 -03:00
Jan Prochazka
87c3b39ae9 v5.1.7-beta.8 2022-12-09 15:50:07 +01:00
Jan Prochazka
a1032138da Merge branch 'summary' 2022-12-09 15:49:22 +01:00
Jan Prochazka
9d6fe2460f fix 2022-12-09 08:47:26 +01:00
44 changed files with 802 additions and 69 deletions

View File

@@ -1,6 +1,6 @@
{
"private": true,
"version": "5.1.7-beta.7",
"version": "5.1.7-beta.8",
"name": "dbgate-all",
"workspaces": [
"packages/*",

View File

@@ -26,7 +26,7 @@
"compare-versions": "^3.6.0",
"cors": "^2.8.5",
"cross-env": "^6.0.3",
"dbgate-query-splitter": "^4.9.2",
"dbgate-query-splitter": "^4.9.3",
"dbgate-sqltree": "^5.0.0-alpha.1",
"dbgate-tools": "^5.0.0-alpha.1",
"debug": "^4.3.4",

View File

@@ -5,6 +5,7 @@ const { archivedir, clearArchiveLinksCache, resolveArchiveFolder } = require('..
const socket = require('../utility/socket');
const { saveFreeTableData } = require('../utility/freeTableStorage');
const loadFilesRecursive = require('../utility/loadFilesRecursive');
const getJslFileName = require('../utility/getJslFileName');
module.exports = {
folders_meta: true,
@@ -150,6 +151,15 @@ module.exports = {
return true;
},
saveJslData_meta: true,
async saveJslData({ folder, file, jslid }) {
const source = getJslFileName(jslid);
const target = path.join(resolveArchiveFolder(folder), `${file}.jsonl`);
await fs.copyFile(source, target);
socket.emitChanged(`archive-files-changed-${folder}`);
return true;
},
async getNewArchiveFolder({ database }) {
const isLink = database.endsWith(database);
const name = isLink ? database.slice(0, -5) : database;

View File

@@ -7,6 +7,7 @@ const DatastoreProxy = require('../utility/DatastoreProxy');
const { saveFreeTableData } = require('../utility/freeTableStorage');
const getJslFileName = require('../utility/getJslFileName');
const JsonLinesDatastore = require('../utility/JsonLinesDatastore');
const requirePluginFunction = require('../utility/requirePluginFunction');
const socket = require('../utility/socket');
function readFirstLine(file) {
@@ -99,10 +100,13 @@ module.exports = {
// return readerInfo;
// },
async ensureDatastore(jslid) {
async ensureDatastore(jslid, formatterFunction) {
let datastore = this.datastores[jslid];
if (!datastore) {
datastore = new JsonLinesDatastore(getJslFileName(jslid));
if (!datastore || datastore.formatterFunction != formatterFunction) {
if (datastore) {
datastore._closeReader();
}
datastore = new JsonLinesDatastore(getJslFileName(jslid), formatterFunction);
// datastore = new DatastoreProxy(getJslFileName(jslid));
this.datastores[jslid] = datastore;
}
@@ -131,8 +135,8 @@ module.exports = {
},
getRows_meta: true,
async getRows({ jslid, offset, limit, filters }) {
const datastore = await this.ensureDatastore(jslid);
async getRows({ jslid, offset, limit, filters, formatterFunction }) {
const datastore = await this.ensureDatastore(jslid, formatterFunction);
return datastore.getRows(offset, limit, _.isEmpty(filters) ? null : filters);
},
@@ -150,8 +154,8 @@ module.exports = {
},
loadFieldValues_meta: true,
async loadFieldValues({ jslid, field, search }) {
const datastore = await this.ensureDatastore(jslid);
async loadFieldValues({ jslid, field, search, formatterFunction }) {
const datastore = await this.ensureDatastore(jslid, formatterFunction);
const res = new Set();
await datastore.enumRows(row => {
if (!filterName(search, row[field])) return true;
@@ -188,4 +192,85 @@ module.exports = {
await fs.promises.writeFile(getJslFileName(jslid), text);
return true;
},
extractTimelineChart_meta: true,
async extractTimelineChart({ jslid, timestampFunction, aggregateFunction, measures }) {
const timestamp = requirePluginFunction(timestampFunction);
const aggregate = requirePluginFunction(aggregateFunction);
const datastore = new JsonLinesDatastore(getJslFileName(jslid));
let mints = null;
let maxts = null;
// pass 1 - counts stats, time range
await datastore.enumRows(row => {
const ts = timestamp(row);
if (!mints || ts < mints) mints = ts;
if (!maxts || ts > maxts) maxts = ts;
return true;
});
const minTime = new Date(mints).getTime();
const maxTime = new Date(maxts).getTime();
const duration = maxTime - minTime;
const STEPS = 100;
let stepCount = duration > 100 * 1000 ? STEPS : Math.round((maxTime - minTime) / 1000);
if (stepCount < 2) {
stepCount = 2;
}
const stepDuration = duration / stepCount;
const labels = _.range(stepCount).map(i => new Date(minTime + stepDuration / 2 + stepDuration * i));
// const datasets = measures.map(m => ({
// label: m.label,
// data: Array(stepCount).fill(0),
// }));
const mproc = measures.map(m => ({
...m,
}));
const data = Array(stepCount)
.fill(0)
.map(() => ({}));
// pass 2 - count measures
await datastore.enumRows(row => {
const ts = timestamp(row);
let part = Math.round((new Date(ts).getTime() - minTime) / stepDuration);
if (part < 0) part = 0;
if (part >= stepCount) part - stepCount - 1;
if (data[part]) {
data[part] = aggregate(data[part], row, stepDuration);
}
return true;
});
datastore._closeReader();
// const measureByField = _.fromPairs(measures.map((m, i) => [m.field, i]));
// for (let mindex = 0; mindex < measures.length; mindex++) {
// for (let stepIndex = 0; stepIndex < stepCount; stepIndex++) {
// const measure = measures[mindex];
// if (measure.perSecond) {
// datasets[mindex].data[stepIndex] /= stepDuration / 1000;
// }
// if (measure.perField) {
// datasets[mindex].data[stepIndex] /= datasets[measureByField[measure.perField]].data[stepIndex];
// }
// }
// }
// for (let i = 0; i < measures.length; i++) {
// if (measures[i].hidden) {
// datasets[i] = null;
// }
// }
return {
labels,
datasets: mproc.map(m => ({
label: m.label,
data: data.map(d => d[m.field] || 0),
})),
};
},
};

View File

@@ -150,6 +150,31 @@ module.exports = {
return true;
},
startProfiler_meta: true,
async startProfiler({ sesid }) {
const jslid = uuidv1();
const session = this.opened.find(x => x.sesid == sesid);
if (!session) {
throw new Error('Invalid session');
}
console.log(`Starting profiler, sesid=${sesid}`);
session.loadingReader_jslid = jslid;
session.subprocess.send({ msgtype: 'startProfiler', jslid });
return { state: 'ok', jslid };
},
stopProfiler_meta: true,
async stopProfiler({ sesid }) {
const session = this.opened.find(x => x.sesid == sesid);
if (!session) {
throw new Error('Invalid session');
}
session.subprocess.send({ msgtype: 'stopProfiler' });
return { state: 'ok' };
},
// cancel_meta: true,
// async cancel({ sesid }) {
// const session = this.opened.find((x) => x.sesid == sesid);

View File

@@ -16,6 +16,7 @@ let storedConnection;
let afterConnectCallbacks = [];
// let currentHandlers = [];
let lastPing = null;
let currentProfiler = null;
class TableWriter {
constructor() {
@@ -210,6 +211,31 @@ function waitConnected() {
});
}
async function handleStartProfiler({ jslid }) {
await waitConnected();
const driver = requireEngineDriver(storedConnection);
if (!allowExecuteCustomScript(driver)) {
process.send({ msgtype: 'done' });
return;
}
const writer = new TableWriter();
writer.initializeFromReader(jslid);
currentProfiler = await driver.startProfiler(systemConnection, {
row: data => writer.rowFromReader(data),
});
currentProfiler.writer = writer;
}
async function handleStopProfiler({ jslid }) {
const driver = requireEngineDriver(storedConnection);
currentProfiler.writer.close();
driver.stopProfiler(systemConnection, currentProfiler);
currentProfiler = null;
}
async function handleExecuteQuery({ sql }) {
await waitConnected();
const driver = requireEngineDriver(storedConnection);
@@ -280,6 +306,8 @@ const messageHandlers = {
connect: handleConnect,
executeQuery: handleExecuteQuery,
executeReader: handleExecuteReader,
startProfiler: handleStartProfiler,
stopProfiler: handleStopProfiler,
ping: handlePing,
// cancel: handleCancel,
};

View File

@@ -3,6 +3,7 @@ const AsyncLock = require('async-lock');
const lock = new AsyncLock();
const stableStringify = require('json-stable-stringify');
const { evaluateCondition } = require('dbgate-sqltree');
const requirePluginFunction = require('./requirePluginFunction');
function fetchNextLineFromReader(reader) {
return new Promise((resolve, reject) => {
@@ -22,14 +23,16 @@ function fetchNextLineFromReader(reader) {
}
class JsonLinesDatastore {
constructor(file) {
constructor(file, formatterFunction) {
this.file = file;
this.formatterFunction = formatterFunction;
this.reader = null;
this.readedDataRowCount = 0;
this.readedSchemaRow = false;
// this.firstRowToBeReturned = null;
this.notifyChangedCallback = null;
this.currentFilter = null;
this.rowFormatter = requirePluginFunction(formatterFunction);
}
_closeReader() {
@@ -62,6 +65,11 @@ class JsonLinesDatastore {
);
}
parseLine(line) {
const res = JSON.parse(line);
return this.rowFormatter ? this.rowFormatter(res) : res;
}
async _readLine(parse) {
// if (this.firstRowToBeReturned) {
// const res = this.firstRowToBeReturned;
@@ -84,14 +92,14 @@ class JsonLinesDatastore {
}
}
if (this.currentFilter) {
const parsedLine = JSON.parse(line);
const parsedLine = this.parseLine(line);
if (evaluateCondition(this.currentFilter, parsedLine)) {
this.readedDataRowCount += 1;
return parse ? parsedLine : true;
}
} else {
this.readedDataRowCount += 1;
return parse ? JSON.parse(line) : true;
return parse ? this.parseLine(line) : true;
}
}

View File

@@ -0,0 +1,16 @@
const _ = require('lodash');
const requirePlugin = require('../shell/requirePlugin');
function requirePluginFunction(functionName) {
if (!functionName) return null;
if (functionName.includes('@')) {
const [shortName, packageName] = functionName.split('@');
const plugin = requirePlugin(packageName);
if (plugin.functions) {
return plugin.functions[shortName];
}
}
return null;
}
module.exports = requirePluginFunction;

View File

@@ -44,6 +44,10 @@ const testCondition = (operator, value) => () => ({
},
});
const multiTestCondition = condition => () => ({
__placeholder__: condition,
});
const compoudCondition = conditionType => conditions => {
if (conditions.length == 1) return conditions[0];
return {
@@ -85,7 +89,15 @@ const createParser = () => {
comma: () => word(','),
not: () => word('NOT'),
empty: () => word('EMPTY'),
array: () => word('ARRAY'),
notExists: r => r.not.then(r.exists).map(testCondition('$exists', false)),
notEmptyArray: r =>
r.not
.then(r.empty)
.then(r.array)
.map(multiTestCondition({ $exists: true, $type: 'array', $ne: [] })),
emptyArray: r => r.empty.then(r.array).map(multiTestCondition({ $exists: true, $eq: [] })),
exists: () => word('EXISTS').map(testCondition('$exists', true)),
true: () => word('TRUE').map(testCondition('$eq', true)),
false: () => word('FALSE').map(testCondition('$eq', false)),
@@ -117,6 +129,8 @@ const createParser = () => {
r.gt,
r.le,
r.ge,
r.notEmptyArray,
r.emptyArray,
r.startsWith,
r.endsWith,
r.contains,

View File

@@ -6,7 +6,7 @@ import { dumpSqlSourceRef } from './dumpSqlSource';
export function evaluateExpression(expr: Expression, values) {
switch (expr.exprType) {
case 'column':
return values[expr.columnName];
return _.get(values, expr.columnName);
case 'placeholder':
return values.__placeholder;

View File

@@ -31,7 +31,7 @@
"typescript": "^4.4.3"
},
"dependencies": {
"dbgate-query-splitter": "^4.9.2",
"dbgate-query-splitter": "^4.9.3",
"dbgate-sqltree": "^5.0.0-alpha.1",
"debug": "^4.3.4",
"json-stable-stringify": "^1.0.1",

View File

@@ -38,25 +38,45 @@ export interface FilterNameDefinition {
// return DoMatch(Filter, value) || camelMatch;
// }
function fuzzysearch(needle, haystack) {
var hlen = haystack.length;
var nlen = needle.length;
if (nlen > hlen) {
return false;
// function fuzzysearch(needle, haystack) {
// var hlen = haystack.length;
// var nlen = needle.length;
// if (nlen > hlen) {
// return false;
// }
// if (nlen === hlen) {
// return needle === haystack;
// }
// outer: for (var i = 0, j = 0; i < nlen; i++) {
// var nch = needle.charCodeAt(i);
// while (j < hlen) {
// if (haystack.charCodeAt(j++) === nch) {
// continue outer;
// }
// }
// return false;
// }
// return true;
// }
function camelMatch(filter: string, text: string): boolean {
if (!text) return false;
if (!filter) return true;
if (filter.replace(/[A-Z]/g, '').length == 0) {
const camelVariants = [text.replace(/[^A-Z]/g, '')];
let s = text,
s0;
do {
s0 = s;
s = s.replace(/([A-Z])([A-Z])([A-Z])/, '$1$3');
} while (s0 != s);
camelVariants.push(s.replace(/[^A-Z]/g, ''));
const camelContains = !!camelVariants.find(x => x.includes(filter.toUpperCase()));
return camelContains;
} else {
return text.toUpperCase().includes(filter.toUpperCase());
}
if (nlen === hlen) {
return needle === haystack;
}
outer: for (var i = 0, j = 0; i < nlen; i++) {
var nch = needle.charCodeAt(i);
while (j < hlen) {
if (haystack.charCodeAt(j++) === nch) {
continue outer;
}
}
return false;
}
return true;
}
export function filterName(filter: string, ...names: (string | FilterNameDefinition)[]) {
@@ -73,13 +93,13 @@ export function filterName(filter: string, ...names: (string | FilterNameDefinit
const namesChild: string[] = namesCompacted.filter(x => x.childName).map(x => x.childName);
for (const token of tokens) {
const tokenUpper = token.toUpperCase();
if (tokenUpper.startsWith('#')) {
const tokenUpperSub = tokenUpper.substring(1);
const found = namesChild.find(name => fuzzysearch(tokenUpperSub, name.toUpperCase()));
// const tokenUpper = token.toUpperCase();
if (token.startsWith('#')) {
// const tokenUpperSub = tokenUpper.substring(1);
const found = namesChild.find(name => camelMatch(token.substring(1), name));
if (!found) return false;
} else {
const found = namesOwn.find(name => fuzzysearch(tokenUpper, name.toUpperCase()));
const found = namesOwn.find(name => camelMatch(token, name));
if (!found) return false;
}
}

View File

@@ -77,6 +77,11 @@ export interface EngineDriver {
supportsDatabaseUrl?: boolean;
supportsDatabaseDump?: boolean;
supportsServerSummary?: boolean;
supportsDatabaseProfiler?: boolean;
profilerFormatterFunction?: string;
profilerTimestampFunction?: string;
profilerChartAggregateFunction?: string;
profilerChartMeasures?: { label: string; field: string }[];
isElectronOnly?: boolean;
supportedCreateDatabase?: boolean;
showConnectionField?: (field: string, values: any) => boolean;
@@ -130,6 +135,8 @@ export interface EngineDriver {
callMethod(pool, method, args);
serverSummary(pool): Promise<ServerSummary>;
summaryCommand(pool, command, row): Promise<void>;
startProfiler(pool, options): Promise<any>;
stopProfiler(pool, profiler): Promise<void>;
analyserClass?: any;
dumperClass?: any;

View File

@@ -24,7 +24,7 @@
"chartjs-adapter-moment": "^1.0.0",
"cross-env": "^7.0.3",
"dbgate-datalib": "^5.0.0-alpha.1",
"dbgate-query-splitter": "^4.9.2",
"dbgate-query-splitter": "^4.9.3",
"dbgate-sqltree": "^5.0.0-alpha.1",
"dbgate-tools": "^5.0.0-alpha.1",
"dbgate-types": "^5.0.0-alpha.1",

View File

@@ -41,7 +41,10 @@
}
export const extractKey = data => data.fileName;
export const createMatcher = ({ fileName }) => filter => filterName(filter, fileName);
export const createMatcher =
({ fileName }) =>
filter =>
filterName(filter, fileName);
const ARCHIVE_ICONS = {
'table.yaml': 'img table',
'view.sql': 'img view',
@@ -67,7 +70,7 @@
import ImportExportModal from '../modals/ImportExportModal.svelte';
import { showModal } from '../modals/modalTools';
import { archiveFilesAsDataSheets, currentArchive, extensions, getCurrentDatabase } from '../stores';
import { archiveFilesAsDataSheets, currentArchive, extensions, getCurrentDatabase, getExtensions } from '../stores';
import createQuickExportMenu from '../utility/createQuickExportMenu';
import { exportQuickExportFile } from '../utility/exportFileTools';
@@ -198,6 +201,29 @@
),
data.fileType.endsWith('.sql') && { text: 'Open SQL', onClick: handleOpenSqlFile },
data.fileType.endsWith('.yaml') && { text: 'Open YAML', onClick: handleOpenYamlFile },
data.fileType == 'jsonl' && {
text: 'Open in profiler',
submenu: getExtensions()
.drivers.filter(eng => eng.profilerFormatterFunction)
.map(eng => ({
text: eng.title,
onClick: () => {
openNewTab({
title: 'Profiler',
icon: 'img profiler',
tabComponent: 'ProfilerTab',
props: {
jslidLoad: `archive://${data.folderName}/${data.fileName}`,
engine: eng.engine,
// profilerFormatterFunction: eng.profilerFormatterFunction,
// profilerTimestampFunction: eng.profilerTimestampFunction,
// profilerChartAggregateFunction: eng.profilerChartAggregateFunction,
// profilerChartMeasures: eng.profilerChartMeasures,
},
});
},
})),
},
];
}
</script>

View File

@@ -254,11 +254,10 @@
text: 'Create database',
onClick: handleCreateDatabase,
},
$openedConnections.includes(data._id) &&
driver?.supportsServerSummary && {
text: 'Server summary',
onClick: handleServerSummary,
},
driver?.supportsServerSummary && {
text: 'Server summary',
onClick: handleServerSummary,
},
],
data.singleDatabase && [
{ divider: true },

View File

@@ -254,6 +254,18 @@
});
};
const handleDatabaseProfiler = () => {
openNewTab({
title: 'Profiler',
icon: 'img profiler',
tabComponent: 'ProfilerTab',
props: {
conid: connection._id,
database: name,
},
});
};
async function handleConfirmSql(sql) {
saveScriptToDatabase({ conid: connection._id, database: name }, sql, false);
}
@@ -284,7 +296,8 @@
!connection.singleDatabase && { onClick: handleDropDatabase, text: 'Drop database' },
{ divider: true },
driver?.databaseEngineTypes?.includes('sql') && { onClick: handleShowDiagram, text: 'Show diagram' },
isSqlOrDoc && { onClick: handleSqlGenerator, text: 'SQL Generator' },
driver?.databaseEngineTypes?.includes('sql') && { onClick: handleSqlGenerator, text: 'SQL Generator' },
driver?.supportsDatabaseProfiler && { onClick: handleDatabaseProfiler, text: 'Database profiler' },
isSqlOrDoc && { onClick: handleOpenJsonModel, text: 'Open model as JSON' },
isSqlOrDoc && { onClick: handleExportModel, text: 'Export DB model - experimental' },
isSqlOrDoc &&

View File

@@ -75,11 +75,17 @@
{:else if value.$oid}
<span class="value">ObjectId("{value.$oid}")</span>
{:else if _.isPlainObject(value)}
<span class="null" title={JSON.stringify(value, undefined, 2)}>(JSON)</span>
{@const svalue = JSON.stringify(value, undefined, 2)}
<span class="null" title={svalue}
>{#if svalue.length < 100}{JSON.stringify(value)}{:else}(JSON){/if}</span
>
{:else if _.isArray(value)}
<span class="null" title={value.map(x => JSON.stringify(x)).join('\n')}>[{value.length} items]</span>
{:else if _.isPlainObject(jsonParsedValue)}
<span class="null" title={JSON.stringify(jsonParsedValue, undefined, 2)}>(JSON)</span>
{@const svalue = JSON.stringify(jsonParsedValue, undefined, 2)}
<span class="null" title={svalue}
>{#if svalue.length < 100}{JSON.stringify(jsonParsedValue)}{:else}(JSON){/if}</span
>
{:else if _.isArray(jsonParsedValue)}
<span class="null" title={jsonParsedValue.map(x => JSON.stringify(x)).join('\n')}
>[{jsonParsedValue.length} items]</span

View File

@@ -235,5 +235,6 @@
bind:loadedRows
bind:selectedCellsPublished
frameSelection={!!macroPreview}
onOpenQuery={openQuery}
{grider}
/>

View File

@@ -33,6 +33,7 @@
export let customCommandIcon = null;
export let onCustomCommand = null;
export let customCommandTooltip = null;
export let formatterFunction = null;
export let pureName = null;
export let schemaName = null;
@@ -168,6 +169,8 @@
{ onClick: () => openFilterWindow('<>'), text: 'Does Not Equal...' },
{ onClick: () => setFilter('EXISTS'), text: 'Field exists' },
{ onClick: () => setFilter('NOT EXISTS'), text: 'Field does not exist' },
{ onClick: () => setFilter('NOT EMPTY ARRAY'), text: 'Array is not empty' },
{ onClick: () => setFilter('EMPTY ARRAY'), text: 'Array is empty' },
{ onClick: () => openFilterWindow('>'), text: 'Greater Than...' },
{ onClick: () => openFilterWindow('>='), text: 'Greater Than Or Equal To...' },
{ onClick: () => openFilterWindow('<'), text: 'Less Than...' },
@@ -274,6 +277,7 @@
schemaName,
pureName,
field: columnName || uniqueName,
formatterFunction,
onConfirm: keys => setFilter(keys.map(x => getFilterValueExpression(x)).join(',')),
});
}

View File

@@ -355,6 +355,7 @@
export let pureName = undefined;
export let schemaName = undefined;
export let allowDefineVirtualReferences = false;
export let formatterFunction;
export let isLoadedAll;
export let loadedTime;
@@ -1634,6 +1635,9 @@
{#if grider.editable}
<FormStyledButton value="Add document" on:click={addJsonDocument} />
{/if}
{#if onOpenQuery}
<FormStyledButton value="Open Query" on:click={onOpenQuery} />
{/if}
</div>
{:else if grider.errors && grider.errors.length > 0}
<div>
@@ -1740,6 +1744,7 @@
{conid}
{database}
{jslid}
{formatterFunction}
driver={display?.driver}
filterType={useEvalFilters ? 'eval' : col.filterType || getFilterType(col.dataType)}
filter={display.getFilter(col.uniqueName)}

View File

@@ -12,12 +12,13 @@
});
async function loadDataPage(props, offset, limit) {
const { jslid, display } = props;
const { jslid, display, formatterFunction } = props;
const response = await apiCall('jsldata/get-rows', {
jslid,
offset,
limit,
formatterFunction,
filters: display ? display.compileFilters() : null,
});
@@ -34,6 +35,9 @@
const response = await apiCall('jsldata/get-stats', { jslid });
return response.rowCount;
}
export let formatterPlugin;
export let formatterFunction;
</script>
<script lang="ts">
@@ -56,6 +60,7 @@
export let jslid;
export let display;
export let formatterFunction;
export const activator = createActivator('JslDataGridCore', false);

View File

@@ -10,10 +10,10 @@ export default class FreeTableGrider extends Grider {
this.model = modelState && modelState.value;
}
getRowData(index: any) {
return this.model.rows[index];
return this.model.rows?.[index];
}
get rowCount() {
return this.model.rows.length;
return this.model.rows?.length;
}
get currentModel(): FreeTableModel {
return this.batchModel || this.model;

View File

@@ -49,6 +49,9 @@
'icon close': 'mdi mdi-close',
'icon unsaved': 'mdi mdi-record',
'icon stop': 'mdi mdi-close-octagon',
'icon play': 'mdi mdi-play',
'icon play-stop': 'mdi mdi-stop',
'icon pause': 'mdi mdi-pause',
'icon filter': 'mdi mdi-filter',
'icon filter-off': 'mdi mdi-filter-off',
'icon reload': 'mdi mdi-reload',
@@ -176,6 +179,7 @@
'img app-command': 'mdi mdi-flash color-icon-green',
'img app-query': 'mdi mdi-view-comfy color-icon-magenta',
'img connection': 'mdi mdi-connection color-icon-blue',
'img profiler': 'mdi mdi-gauge color-icon-blue',
'img add': 'mdi mdi-plus-circle color-icon-green',
'img minus': 'mdi mdi-minus-circle color-icon-red',

View File

@@ -25,6 +25,7 @@
export let driver;
export let multiselect = false;
export let jslid;
export let formatterFunction;
// console.log('ValueLookupModal', conid, database, pureName, schemaName, columnName, driver);
@@ -42,6 +43,7 @@
jslid,
search,
field,
formatterFunction,
});
} else {
rows = await apiCall('database-connections/load-field-values', {

View File

@@ -447,6 +447,7 @@
editor.container.addEventListener('contextmenu', handleContextMenu);
editor.keyBinding.addKeyboardHandler(handleKeyDown);
editor.renderer.setScrollMargin(2, 0);
changedQueryParts();
// editor.session.addGutterDecoration(0, 'ace-gutter-sql-run');

View File

@@ -216,7 +216,7 @@ export const getCurrentDatabase = () => currentDatabaseValue;
let currentSettingsValue = null;
export const getCurrentSettings = () => currentSettingsValue || {};
let extensionsValue = null;
let extensionsValue: ExtensionsDirectory = null;
extensions.subscribe(value => {
extensionsValue = value;
});

View File

@@ -0,0 +1,250 @@
<script lang="ts" context="module">
const getCurrentEditor = () => getActiveComponent('ProfilerTab');
registerCommand({
id: 'profiler.start',
category: 'Profiler',
name: 'Start profiling',
icon: 'icon play',
testEnabled: () => getCurrentEditor()?.startProfilingEnabled(),
onClick: () => getCurrentEditor().startProfiling(),
});
registerCommand({
id: 'profiler.stop',
category: 'Profiler',
name: 'Stop profiling',
icon: 'icon play-stop',
testEnabled: () => getCurrentEditor()?.stopProfilingEnabled(),
onClick: () => getCurrentEditor().stopProfiling(),
});
registerCommand({
id: 'profiler.save',
category: 'Profiler',
name: 'Save',
icon: 'icon save',
testEnabled: () => getCurrentEditor()?.saveEnabled(),
onClick: () => getCurrentEditor().save(),
});
</script>
<script>
import { findEngineDriver } from 'dbgate-tools';
import { onDestroy, onMount } from 'svelte';
import ToolStripCommandButton from '../buttons/ToolStripCommandButton.svelte';
import ToolStripContainer from '../buttons/ToolStripContainer.svelte';
import invalidateCommands from '../commands/invalidateCommands';
import registerCommand from '../commands/registerCommand';
import JslDataGrid from '../datagrid/JslDataGrid.svelte';
import ErrorInfo from '../elements/ErrorInfo.svelte';
import VerticalSplitter from '../elements/VerticalSplitter.svelte';
import { showModal } from '../modals/modalTools';
import SaveArchiveModal from '../modals/SaveArchiveModal.svelte';
import { currentArchive, selectedWidget } from '../stores';
import { apiCall } from '../utility/api';
import createActivator, { getActiveComponent } from '../utility/createActivator';
import { useConnectionInfo } from '../utility/metadataLoaders';
import { extensions } from '../stores';
import ChartCore from '../charts/ChartCore.svelte';
import LoadingInfo from '../elements/LoadingInfo.svelte';
import randomcolor from 'randomcolor';
export const activator = createActivator('ProfilerTab', true);
export let conid;
export let database;
export let engine;
export let jslidLoad;
let jslidSession;
let isProfiling = false;
let sessionId;
let isLoadingChart = false;
let intervalId;
let chartData;
$: connection = useConnectionInfo({ conid });
$: driver = findEngineDriver(engine || $connection, $extensions);
$: jslid = jslidSession || jslidLoad;
onMount(() => {
intervalId = setInterval(() => {
if (sessionId) {
apiCall('sessions/ping', {
sesid: sessionId,
});
}
}, 15 * 1000);
});
$: {
if (jslidLoad && driver) {
loadChart();
}
}
onDestroy(() => {
clearInterval(intervalId);
});
export async function startProfiling() {
isProfiling = true;
let sesid = sessionId;
if (!sesid) {
const resp = await apiCall('sessions/create', {
conid,
database,
});
sesid = resp.sesid;
sessionId = sesid;
}
const resp = await apiCall('sessions/start-profiler', {
sesid,
});
jslidSession = resp.jslid;
invalidateCommands();
}
export function startProfilingEnabled() {
return conid && database && !isProfiling;
}
async function loadChart() {
isLoadingChart = true;
const colors = randomcolor({
count: driver.profilerChartMeasures.length,
seed: 5,
});
const data = await apiCall('jsldata/extract-timeline-chart', {
jslid,
timestampFunction: driver.profilerTimestampFunction,
aggregateFunction: driver.profilerChartAggregateFunction,
measures: driver.profilerChartMeasures,
});
chartData = {
...data,
labels: data.labels.map(x => new Date(x)),
datasets: data.datasets.map((x, i) => ({
...x,
borderColor: colors[i],
})),
};
isLoadingChart = false;
}
export async function stopProfiling() {
isProfiling = false;
await apiCall('sessions/stop-profiler', { sesid: sessionId });
await apiCall('sessions/kill', { sesid: sessionId });
sessionId = null;
invalidateCommands();
loadChart();
}
export function stopProfilingEnabled() {
return conid && database && isProfiling;
}
export function saveEnabled() {
return !!jslidSession;
}
async function doSave(folder, file) {
await apiCall('archive/save-jsl-data', { folder, file, jslid });
currentArchive.set(folder);
selectedWidget.set('archive');
}
export function save() {
showModal(SaveArchiveModal, {
// folder: archiveFolder,
// file: archiveFile,
onSave: doSave,
});
}
// const data = [
// { year: 2010, count: 10 },
// { year: 2011, count: 20 },
// { year: 2012, count: 15 },
// { year: 2013, count: 25 },
// { year: 2014, count: 22 },
// { year: 2015, count: 30 },
// { year: 2016, count: 28 },
// ];
// {
// labels: data.map(row => row.year),
// datasets: [
// {
// label: 'Acquisitions by year',
// data: data.map(row => row.count),
// },
// ],
// }
</script>
<ToolStripContainer>
{#if jslid}
<VerticalSplitter allowCollapseChild1 allowCollapseChild2>
<svelte:fragment slot="1">
{#key jslid}
<JslDataGrid {jslid} listenInitializeFile formatterFunction={driver?.profilerFormatterFunction} />
{/key}
</svelte:fragment>
<svelte:fragment slot="2">
{#if isLoadingChart}
<LoadingInfo wrapper message="Loading chart" />
{:else}
<ChartCore
title="Profile data"
data={chartData}
options={{
maintainAspectRatio: false,
scales: {
x: {
type: 'time',
distribution: 'linear',
time: {
tooltipFormat: 'D. M. YYYY HH:mm',
displayFormats: {
millisecond: 'HH:mm:ss.SSS',
second: 'HH:mm:ss',
minute: 'HH:mm',
hour: 'D.M hA',
day: 'D. M.',
week: 'D. M. YYYY',
month: 'MM-YYYY',
quarter: '[Q]Q - YYYY',
year: 'YYYY',
},
},
},
},
}}
/>
{/if}
</svelte:fragment>
</VerticalSplitter>
{:else}
<ErrorInfo message="Profiler not yet started" alignTop />
{/if}
<svelte:fragment slot="toolstrip">
<ToolStripCommandButton command="profiler.start" />
<ToolStripCommandButton command="profiler.stop" />
<ToolStripCommandButton command="profiler.save" />
</svelte:fragment>
</ToolStripContainer>

View File

@@ -10,6 +10,7 @@
isRelatedToTab: true,
icon: 'icon reload',
onClick: () => getCurrentEditor().refresh(),
testEnabled: () => getCurrentEditor() != null,
});
</script>
@@ -37,7 +38,7 @@
}
async function runAction(action, row) {
const { command, openQuery } = action;
const { command, openQuery, openTab, addDbProps } = action;
if (command) {
await apiCall('server-connections/summary-command', { conid, refreshToken, command, row });
refresh();
@@ -54,6 +55,20 @@
},
});
}
if (openTab) {
const props = {};
if (addDbProps) {
props['conid'] = conid;
props['database'] = row.name;
}
openNewTab({
...openTab,
props: {
...openTab.props,
...props,
},
});
}
}
</script>
@@ -76,7 +91,7 @@
<svelte:fragment slot="2" let:row let:col>
{#each col.actions as action, index}
{#if index > 0}
<span> | </span>
<span class="action-separator">|</span>
{/if}
<Link onClick={() => runAction(action, row)}>{action.header}</Link>
{/each}
@@ -100,4 +115,8 @@
background-color: var(--theme-bg-0);
overflow: auto;
}
.action-separator {
margin: 0 5px;
}
</style>

View File

@@ -27,6 +27,7 @@ import * as ConnectionTab from './ConnectionTab.svelte';
import * as MapTab from './MapTab.svelte';
import * as PerspectiveTab from './PerspectiveTab.svelte';
import * as ServerSummaryTab from './ServerSummaryTab.svelte';
import * as ProfilerTab from './ProfilerTab.svelte';
export default {
TableDataTab,
@@ -58,4 +59,5 @@ export default {
MapTab,
PerspectiveTab,
ServerSummaryTab,
ProfilerTab,
};

View File

@@ -57,7 +57,7 @@ export default {
}),
},
{
label: 'CSV file (semicolor separated)',
label: 'CSV file (semicolon separated)',
extension: 'csv',
createWriter: (fileName) => ({
functionName: 'writer@dbgate-plugin-csv',

View File

@@ -32,7 +32,7 @@
},
"devDependencies": {
"dbgate-plugin-tools": "^1.0.7",
"dbgate-query-splitter": "^4.9.2",
"dbgate-query-splitter": "^4.9.3",
"webpack": "^4.42.0",
"webpack-cli": "^3.3.11",
"dbgate-tools": "^5.0.0-alpha.1",

View File

@@ -38,7 +38,7 @@ async function getScriptableDb(pool) {
const db = pool.__getDatabase();
const collections = await db.listCollections().toArray();
for (const collection of collections) {
db[collection.name] = db.collection(collection.name);
_.set(db, collection.name, db.collection(collection.name));
}
return db;
}
@@ -165,6 +165,49 @@ const driver = {
options.done();
},
async startProfiler(pool, options) {
const db = await getScriptableDb(pool);
const old = await db.command({ profile: -1 });
await db.command({ profile: 2 });
const cursor = await db.collection('system.profile').find({
ns: /^((?!(admin\.\$cmd|\.system|\.tmp\.)).)*$/,
ts: { $gt: new Date() },
'command.profile': { $exists: false },
'command.collStats': { $exists: false },
'command.collstats': { $exists: false },
'command.createIndexes': { $exists: false },
'command.listIndexes': { $exists: false },
// "command.cursor": {"$exists": false},
'command.create': { $exists: false },
'command.dbstats': { $exists: false },
'command.scale': { $exists: false },
'command.explain': { $exists: false },
'command.killCursors': { $exists: false },
'command.count': { $ne: 'system.profile' },
op: /^((?!(getmore|killcursors)).)/i,
});
cursor.addCursorFlag('tailable', true);
cursor.addCursorFlag('awaitData', true);
cursor
.forEach((row) => {
// console.log('ROW', row);
options.row(row);
})
.catch((err) => {
console.error('Cursor stopped with error:', err.message);
});
return {
cursor,
old,
};
},
async stopProfiler(pool, { cursor, old }) {
cursor.close();
const db = await getScriptableDb(pool);
await db.command({ profile: old.was, slowms: old.slowms });
},
async readQuery(pool, sql, structure) {
try {
const json = JSON.parse(sql);
@@ -417,10 +460,22 @@ const driver = {
header: 'All',
command: 'profileAll',
},
// {
// header: 'View',
// openQuery: "db['system.profile'].find()",
// tabTitle: 'Profile data',
// },
{
header: 'View',
openQuery: "db['system.profile'].find()",
tabTitle: 'Profile data',
openTab: {
title: 'system.profile',
icon: 'img collection',
tabComponent: 'CollectionDataTab',
props: {
pureName: 'system.profile',
},
},
addDbProps: true,
},
],
},

View File

@@ -1,6 +1,16 @@
const driver = require('./driver');
const {
formatProfilerEntry,
extractProfileTimestamp,
aggregateProfileChartEntry,
} = require('../frontend/profilerFunctions');
module.exports = {
packageName: 'dbgate-plugin-mongo',
drivers: [driver],
functions: {
formatProfilerEntry,
extractProfileTimestamp,
aggregateProfileChartEntry,
},
};

View File

@@ -33,6 +33,15 @@ const driver = {
defaultPort: 27017,
supportsDatabaseUrl: true,
supportsServerSummary: true,
supportsDatabaseProfiler: true,
profilerFormatterFunction: 'formatProfilerEntry@dbgate-plugin-mongo',
profilerTimestampFunction: 'extractProfileTimestamp@dbgate-plugin-mongo',
profilerChartAggregateFunction: 'aggregateProfileChartEntry@dbgate-plugin-mongo',
profilerChartMeasures: [
{ label: 'Req count/s', field: 'countPerSec' },
{ label: 'Avg duration', field: 'avgDuration' },
{ label: 'Max duration', field: 'maxDuration' },
],
databaseUrlPlaceholder: 'e.g. mongodb://username:password@mongodb.mydomain.net/dbname',
getQuerySplitterOptions: () => mongoSplitterOptions,

View File

@@ -1,6 +1,12 @@
import driver from './driver';
import { formatProfilerEntry, extractProfileTimestamp, aggregateProfileChartEntry } from './profilerFunctions';
export default {
packageName: 'dbgate-plugin-mongo',
drivers: [driver],
functions: {
formatProfilerEntry,
extractProfileTimestamp,
aggregateProfileChartEntry,
},
};

View File

@@ -0,0 +1,103 @@
const _ = require('lodash');
function formatProfilerEntry(obj) {
const ts = obj.ts;
const stats = { millis: obj.millis };
let op = obj.op;
let doc;
let query;
let ext;
if (op == 'query') {
const cmd = obj.command || obj.query;
doc = cmd.find;
query = cmd.filter;
ext = _.pick(cmd, ['sort', 'limit', 'skip']);
} else if (op == 'update') {
doc = obj.ns.split('.').slice(-1)[0];
query = obj.command && obj.command.q;
ext = _.pick(obj, ['nModified', 'nMatched']);
} else if (op == 'insert') {
doc = obj.ns.split('.').slice(-1)[0];
ext = _.pick(obj, ['ninserted']);
} else if (op == 'remove') {
doc = obj.ns.split('.').slice(-1)[0];
query = obj.command && obj.command.q;
} else if (op == 'command' && obj.command) {
const cmd = obj.command;
if (cmd.count) {
op = 'count';
query = cmd.query;
} else if (cmd.aggregate) {
op = 'aggregate';
query = cmd.pipeline;
} else if (cmd.distinct) {
op = 'distinct';
query = cmd.query;
ext = _.pick(cmd, ['key']);
} else if (cmd.drop) {
op = 'drop';
} else if (cmd.findandmodify) {
op = 'findandmodify';
query = cmd.query;
ext = _.pick(cmd, ['sort', 'update', 'remove', 'fields', 'upsert', 'new']);
} else if (cmd.group) {
op = 'group';
doc = cmd.group.ns;
ext = _.pick(cmd, ['key', 'initial', 'cond', '$keyf', '$reduce', 'finalize']);
} else if (cmd.map) {
op = 'map';
doc = cmd.mapreduce;
query = _.omit(cmd, ['mapreduce', 'map', 'reduce']);
ext = { map: cmd.map, reduce: cmd.reduce };
} else {
// unknown command
op = 'unknown';
query = obj;
}
} else {
// unknown operation
query = obj;
}
return {
ts,
op,
doc,
query,
ext,
stats,
};
}
function extractProfileTimestamp(obj) {
return obj.ts;
}
function aggregateProfileChartEntry(aggr, obj, stepDuration) {
// const fmt = formatProfilerEntry(obj);
const countAll = (aggr.countAll || 0) + 1;
const sumMillis = (aggr.sumMillis || 0) + obj.millis;
const maxDuration = obj.millis > (aggr.maxDuration || 0) ? obj.millis : aggr.maxDuration || 0;
return {
countAll,
sumMillis,
countPerSec: (countAll / stepDuration) * 1000,
avgDuration: sumMillis / countAll,
maxDuration,
};
// return {
// ts: fmt.ts,
// millis: fmt.stats.millis,
// countAll: 1,
// countPerSec: 1,
// };
}
module.exports = {
formatProfilerEntry,
extractProfileTimestamp,
aggregateProfileChartEntry,
};

View File

@@ -32,7 +32,7 @@
},
"devDependencies": {
"dbgate-plugin-tools": "^1.0.7",
"dbgate-query-splitter": "^4.9.2",
"dbgate-query-splitter": "^4.9.3",
"webpack": "^4.42.0",
"webpack-cli": "^3.3.11",
"dbgate-tools": "^5.0.0-alpha.1",

View File

@@ -33,7 +33,7 @@
"devDependencies": {
"antares-mysql-dumper": "^0.0.1",
"dbgate-plugin-tools": "^1.0.7",
"dbgate-query-splitter": "^4.9.2",
"dbgate-query-splitter": "^4.9.3",
"dbgate-tools": "^5.0.0-alpha.1",
"mysql2": "^2.3.3",
"webpack": "^4.42.0",

View File

@@ -4,5 +4,5 @@ select
TABLE_ROWS as tableRowCount,
case when ENGINE='InnoDB' then CREATE_TIME else coalesce(UPDATE_TIME, CREATE_TIME) end as modifyDate
from information_schema.tables
where TABLE_SCHEMA = '#DATABASE#' and TABLE_TYPE='BASE TABLE' and TABLE_NAME =OBJECT_ID_CONDITION;
where TABLE_SCHEMA = '#DATABASE#' and (TABLE_TYPE='BASE TABLE' or TABLE_TYPE='SYSTEM VERSIONED') and TABLE_NAME =OBJECT_ID_CONDITION;
`;

View File

@@ -31,7 +31,7 @@
},
"devDependencies": {
"dbgate-plugin-tools": "^1.0.7",
"dbgate-query-splitter": "^4.9.2",
"dbgate-query-splitter": "^4.9.3",
"dbgate-tools": "^5.0.0-alpha.1",
"lodash": "^4.17.21",
"pg": "^8.7.1",

View File

@@ -30,7 +30,7 @@
},
"devDependencies": {
"dbgate-plugin-tools": "^1.0.7",
"dbgate-query-splitter": "^4.9.2",
"dbgate-query-splitter": "^4.9.3",
"dbgate-tools": "^5.0.0-alpha.1",
"lodash": "^4.17.21",
"webpack": "^4.42.0",

View File

@@ -32,7 +32,7 @@
"devDependencies": {
"dbgate-tools": "^5.0.0-alpha.1",
"dbgate-plugin-tools": "^1.0.4",
"dbgate-query-splitter": "^4.9.2",
"dbgate-query-splitter": "^4.9.3",
"byline": "^5.0.0",
"webpack": "^4.42.0",
"webpack-cli": "^3.3.11"

View File

@@ -3331,10 +3331,10 @@ dbgate-plugin-xml@^5.0.0-alpha.1:
resolved "https://registry.yarnpkg.com/dbgate-plugin-xml/-/dbgate-plugin-xml-5.0.9.tgz#c3abf6ed8cd1450c45058d35c9326458833ed27e"
integrity sha512-P8Em1A6HhF0BfxEDDEUyzdgFeJHEC5vbg12frANpWHjO3V1HGdygsT2z1ukLK8FS5BLW/vcCdOFldXZGh+wWvg==
dbgate-query-splitter@^4.9.2:
version "4.9.2"
resolved "https://registry.yarnpkg.com/dbgate-query-splitter/-/dbgate-query-splitter-4.9.2.tgz#ab1a60e60887ca750dd263a59db66e82c6461a46"
integrity sha512-MwZzNNLILdUv8rg6mFysLizIEdZsLLHEOL4lAHrvLPtHaLOAb275ogtgieLqjcnsXkPlV03i2t1b697aQYdfLQ==
dbgate-query-splitter@^4.9.3:
version "4.9.3"
resolved "https://registry.yarnpkg.com/dbgate-query-splitter/-/dbgate-query-splitter-4.9.3.tgz#f66396da9ae3cc8f775a282143bfca3441248aa2"
integrity sha512-QMppAy3S6NGQMawNokmhbpZURvLCETyu/8yTfqWUHGdlK963fdSpmoX1A+9SjCDp62sX0vYntfD7uzd6jVSRcw==
debug@2.6.9, debug@^2.2.0, debug@^2.3.3:
version "2.6.9"