From 1c1431014cb888013f4180b5298ca7f1c4a4255b Mon Sep 17 00:00:00 2001 From: Stela Augustinova <82719799+Stelinkaa@users.noreply.github.com> Date: Wed, 1 Apr 2026 15:50:20 +0200 Subject: [PATCH] SYNC: Merge pull request #87 from dbgate/feature/collection-test --- .../__tests__/collection.spec.js | 536 ++++++++++++++++++ integration-tests/docker-compose.yaml | 17 + integration-tests/engines.js | 23 + integration-tests/wait.js | 6 +- workflow-templates/run-tests.yaml | 15 + 5 files changed, 595 insertions(+), 2 deletions(-) create mode 100644 integration-tests/__tests__/collection.spec.js diff --git a/integration-tests/__tests__/collection.spec.js b/integration-tests/__tests__/collection.spec.js new file mode 100644 index 000000000..75abeb1e1 --- /dev/null +++ b/integration-tests/__tests__/collection.spec.js @@ -0,0 +1,536 @@ +const requireEngineDriver = require('dbgate-api/src/utility/requireEngineDriver'); +const crypto = require('crypto'); +const stream = require('stream'); +const { mongoDbEngine, dynamoDbEngine } = require('../engines'); +const tableWriter = require('dbgate-api/src/shell/tableWriter'); +const tableReader = require('dbgate-api/src/shell/tableReader'); +const copyStream = require('dbgate-api/src/shell/copyStream'); + +function randomCollectionName() { + return 'test_' + crypto.randomBytes(6).toString('hex'); +} + +const documentEngines = [ + { label: 'MongoDB', engine: mongoDbEngine }, + { label: 'DynamoDB', engine: dynamoDbEngine }, +]; + +async function connectEngine(engine) { + const driver = requireEngineDriver(engine.connection); + const conn = await driver.connect(engine.connection); + return { driver, conn }; +} + +async function createCollection(driver, conn, collectionName, engine) { + if (engine.connection.engine.startsWith('dynamodb')) { + await driver.operation(conn, { + type: 'createCollection', + collection: { + name: collectionName, + partitionKey: '_id', + partitionKeyType: 'S', + }, + }); + } else { + await driver.operation(conn, { + type: 'createCollection', + collection: { name: collectionName }, + }); + } +} + +async function dropCollection(driver, conn, collectionName) { + try { + await driver.operation(conn, { + type: 'dropCollection', + collection: collectionName, + }); + } catch (e) { + // Ignore errors when dropping (collection may not exist) + } +} + +async function insertDocument(driver, conn, collectionName, doc) { + return driver.updateCollection(conn, { + inserts: [{ pureName: collectionName, document: {}, fields: doc }], + updates: [], + deletes: [], + }); +} + +async function readAll(driver, conn, collectionName) { + return driver.readCollection(conn, { pureName: collectionName, limit: 1000 }); +} + +async function updateDocument(driver, conn, collectionName, condition, fields) { + return driver.updateCollection(conn, { + inserts: [], + updates: [{ pureName: collectionName, condition, fields }], + deletes: [], + }); +} + +async function deleteDocument(driver, conn, collectionName, condition) { + return driver.updateCollection(conn, { + inserts: [], + updates: [], + deletes: [{ pureName: collectionName, condition }], + }); +} + +describe('Collection CRUD', () => { + describe.each(documentEngines.map(e => [e.label, e.engine]))('%s', (label, engine) => { + let driver; + let conn; + let collectionName; + + beforeAll(async () => { + const result = await connectEngine(engine); + driver = result.driver; + conn = result.conn; + }); + + afterAll(async () => { + if (conn) { + await driver.close(conn); + } + }); + + beforeEach(async () => { + collectionName = randomCollectionName(); + await createCollection(driver, conn, collectionName, engine); + }); + + afterEach(async () => { + await dropCollection(driver, conn, collectionName); + }); + + // ---- INSERT ---- + + test('insert a single document', async () => { + const res = await insertDocument(driver, conn, collectionName, { + _id: 'doc1', + name: 'Alice', + age: 30, + }); + expect(res.inserted.length).toBe(1); + + const all = await readAll(driver, conn, collectionName); + expect(all.rows.length).toBe(1); + expect(all.rows[0].name).toBe('Alice'); + expect(all.rows[0].age).toBe(30); + }); + + test('insert multiple documents', async () => { + await insertDocument(driver, conn, collectionName, { _id: 'a1', name: 'Alice' }); + await insertDocument(driver, conn, collectionName, { _id: 'a2', name: 'Bob' }); + await insertDocument(driver, conn, collectionName, { _id: 'a3', name: 'Charlie' }); + + const all = await readAll(driver, conn, collectionName); + expect(all.rows.length).toBe(3); + const names = all.rows.map(r => r.name).sort(); + expect(names).toEqual(['Alice', 'Bob', 'Charlie']); + }); + + test('insert document with nested object', async () => { + await insertDocument(driver, conn, collectionName, { + _id: 'nested1', + name: 'Alice', + address: { city: 'Prague', zip: '11000' }, + }); + + const all = await readAll(driver, conn, collectionName); + expect(all.rows.length).toBe(1); + expect(all.rows[0].address.city).toBe('Prague'); + expect(all.rows[0].address.zip).toBe('11000'); + }); + + // ---- READ ---- + + test('read from empty collection returns no rows', async () => { + const all = await readAll(driver, conn, collectionName); + expect(all.rows.length).toBe(0); + }); + + test('read with limit', async () => { + await insertDocument(driver, conn, collectionName, { _id: 'l1', name: 'A' }); + await insertDocument(driver, conn, collectionName, { _id: 'l2', name: 'B' }); + await insertDocument(driver, conn, collectionName, { _id: 'l3', name: 'C' }); + + const limited = await driver.readCollection(conn, { + pureName: collectionName, + limit: 2, + }); + expect(limited.rows.length).toBe(2); + }); + + test('count documents', async () => { + await insertDocument(driver, conn, collectionName, { _id: 'c1', name: 'A' }); + await insertDocument(driver, conn, collectionName, { _id: 'c2', name: 'B' }); + + const result = await driver.readCollection(conn, { + pureName: collectionName, + countDocuments: true, + }); + expect(result.count).toBe(2); + }); + + test('count documents on empty collection returns zero', async () => { + const result = await driver.readCollection(conn, { + pureName: collectionName, + countDocuments: true, + }); + expect(result.count).toBe(0); + }); + + // ---- UPDATE ---- + + test('update an existing document', async () => { + await insertDocument(driver, conn, collectionName, { _id: 'u1', name: 'Alice', age: 25 }); + + const res = await updateDocument(driver, conn, collectionName, { _id: 'u1' }, { name: 'Alice Updated' }); + expect(res.errorMessage).toBeUndefined(); + + const all = await readAll(driver, conn, collectionName); + expect(all.rows.length).toBe(1); + expect(all.rows[0].name).toBe('Alice Updated'); + }); + + test('update does not create new document', async () => { + await insertDocument(driver, conn, collectionName, { _id: 'u2', name: 'Bob' }); + + await updateDocument(driver, conn, collectionName, { _id: 'nonexistent' }, { name: 'Ghost' }); + + const all = await readAll(driver, conn, collectionName); + expect(all.rows.length).toBe(1); + expect(all.rows[0].name).toBe('Bob'); + }); + + test('update only specified fields', async () => { + await insertDocument(driver, conn, collectionName, { _id: 'u3', name: 'Carol', age: 40, city: 'London' }); + + await updateDocument(driver, conn, collectionName, { _id: 'u3' }, { age: 41 }); + + const all = await readAll(driver, conn, collectionName); + expect(all.rows.length).toBe(1); + expect(all.rows[0].name).toBe('Carol'); + expect(all.rows[0].age).toBe(41); + expect(all.rows[0].city).toBe('London'); + }); + + // ---- DELETE ---- + + test('delete an existing document', async () => { + await insertDocument(driver, conn, collectionName, { _id: 'd1', name: 'Alice' }); + await insertDocument(driver, conn, collectionName, { _id: 'd2', name: 'Bob' }); + + const res = await deleteDocument(driver, conn, collectionName, { _id: 'd1' }); + expect(res.errorMessage).toBeUndefined(); + + const all = await readAll(driver, conn, collectionName); + expect(all.rows.length).toBe(1); + expect(all.rows[0].name).toBe('Bob'); + }); + + test('delete non-existing document does not affect collection', async () => { + await insertDocument(driver, conn, collectionName, { _id: 'dx1', name: 'Alice' }); + + await deleteDocument(driver, conn, collectionName, { _id: 'nonexistent' }); + + const all = await readAll(driver, conn, collectionName); + expect(all.rows.length).toBe(1); + expect(all.rows[0].name).toBe('Alice'); + }); + + test('delete all documents leaves empty collection', async () => { + await insertDocument(driver, conn, collectionName, { _id: 'da1', name: 'A' }); + await insertDocument(driver, conn, collectionName, { _id: 'da2', name: 'B' }); + + await deleteDocument(driver, conn, collectionName, { _id: 'da1' }); + await deleteDocument(driver, conn, collectionName, { _id: 'da2' }); + + const all = await readAll(driver, conn, collectionName); + expect(all.rows.length).toBe(0); + }); + + // ---- EDGE CASES ---- + + test('insert and read document with empty string field', async () => { + await insertDocument(driver, conn, collectionName, { _id: 'e1', name: '', value: 'test' }); + + const all = await readAll(driver, conn, collectionName); + expect(all.rows.length).toBe(1); + expect(all.rows[0].name).toBe(''); + expect(all.rows[0].value).toBe('test'); + }); + + test('insert and read document with numeric values', async () => { + await insertDocument(driver, conn, collectionName, { + _id: 'n1', + intVal: 42, + floatVal: 3.14, + zero: 0, + negative: -10, + }); + + const all = await readAll(driver, conn, collectionName); + expect(all.rows.length).toBe(1); + expect(all.rows[0].intVal).toBe(42); + expect(all.rows[0].floatVal).toBeCloseTo(3.14); + expect(all.rows[0].zero).toBe(0); + expect(all.rows[0].negative).toBe(-10); + }); + + test('insert and read document with boolean values', async () => { + await insertDocument(driver, conn, collectionName, { + _id: 'b1', + active: true, + deleted: false, + }); + + const all = await readAll(driver, conn, collectionName); + expect(all.rows.length).toBe(1); + expect(all.rows[0].active).toBe(true); + expect(all.rows[0].deleted).toBe(false); + }); + + test('reading non-existing collection returns error or empty', async () => { + const result = await driver.readCollection(conn, { + pureName: 'nonexistent_collection_' + crypto.randomBytes(4).toString('hex'), + limit: 10, + }); + // Depending on the driver, this may return an error or empty rows + if (result.errorMessage) { + expect(typeof result.errorMessage).toBe('string'); + } else { + expect(result.rows.length).toBe(0); + } + }); + + test('replace full document via update with document field', async () => { + await insertDocument(driver, conn, collectionName, { _id: 'r1', name: 'Original', extra: 'data' }); + + await driver.updateCollection(conn, { + inserts: [], + updates: [ + { + pureName: collectionName, + condition: { _id: 'r1' }, + document: { _id: 'r1', name: 'Replaced' }, + fields: {}, + }, + ], + deletes: [], + }); + + const all = await readAll(driver, conn, collectionName); + expect(all.rows.length).toBe(1); + expect(all.rows[0].name).toBe('Replaced'); + }); + + test('insert then update then delete lifecycle', async () => { + // Insert + await insertDocument(driver, conn, collectionName, { _id: 'life1', name: 'Lifecycle', status: 'created' }); + let all = await readAll(driver, conn, collectionName); + expect(all.rows.length).toBe(1); + expect(all.rows[0].status).toBe('created'); + + // Update + await updateDocument(driver, conn, collectionName, { _id: 'life1' }, { status: 'updated' }); + all = await readAll(driver, conn, collectionName); + expect(all.rows[0].status).toBe('updated'); + + // Delete + await deleteDocument(driver, conn, collectionName, { _id: 'life1' }); + all = await readAll(driver, conn, collectionName); + expect(all.rows.length).toBe(0); + }); + }); +}); + +function createDocumentImportStream(documents) { + const pass = new stream.PassThrough({ objectMode: true }); + pass.write({ __isStreamHeader: true, __isDynamicStructure: true }); + for (const doc of documents) { + pass.write(doc); + } + pass.end(); + return pass; +} + +function createExportStream() { + const writable = new stream.Writable({ objectMode: true }); + writable.resultArray = []; + writable._write = (chunk, encoding, callback) => { + writable.resultArray.push(chunk); + callback(); + }; + return writable; +} + +describe('Collection Import/Export', () => { + describe.each(documentEngines.map(e => [e.label, e.engine]))('%s', (label, engine) => { + let driver; + let conn; + let collectionName; + + beforeAll(async () => { + const result = await connectEngine(engine); + driver = result.driver; + conn = result.conn; + }); + + afterAll(async () => { + if (conn) { + await driver.close(conn); + } + }); + + beforeEach(async () => { + collectionName = randomCollectionName(); + await createCollection(driver, conn, collectionName, engine); + }); + + afterEach(async () => { + await dropCollection(driver, conn, collectionName); + }); + + test('import documents via stream', async () => { + const documents = [ + { _id: 'imp1', name: 'Alice', age: 30 }, + { _id: 'imp2', name: 'Bob', age: 25 }, + { _id: 'imp3', name: 'Charlie', age: 35 }, + ]; + + const reader = createDocumentImportStream(documents); + const writer = await tableWriter({ + systemConnection: conn, + driver, + pureName: collectionName, + createIfNotExists: true, + }); + await copyStream(reader, writer); + + const all = await readAll(driver, conn, collectionName); + expect(all.rows.length).toBe(3); + const names = all.rows.map(r => r.name).sort(); + expect(names).toEqual(['Alice', 'Bob', 'Charlie']); + }); + + test('export documents via stream', async () => { + await insertDocument(driver, conn, collectionName, { _id: 'exp1', name: 'Alice', city: 'Prague' }); + await insertDocument(driver, conn, collectionName, { _id: 'exp2', name: 'Bob', city: 'Vienna' }); + await insertDocument(driver, conn, collectionName, { _id: 'exp3', name: 'Charlie', city: 'Berlin' }); + + const reader = await tableReader({ + systemConnection: conn, + driver, + pureName: collectionName, + }); + const writer = createExportStream(); + await copyStream(reader, writer); + + const rows = writer.resultArray.filter(x => !x.__isStreamHeader); + expect(rows.length).toBe(3); + const names = rows.map(r => r.name).sort(); + expect(names).toEqual(['Alice', 'Bob', 'Charlie']); + }); + + test('import then export round-trip', async () => { + const documents = [ + { _id: 'rt1', name: 'Alice', value: 100 }, + { _id: 'rt2', name: 'Bob', value: 200 }, + { _id: 'rt3', name: 'Charlie', value: 300 }, + { _id: 'rt4', name: 'Diana', value: 400 }, + ]; + + // Import + const importReader = createDocumentImportStream(documents); + const importWriter = await tableWriter({ + systemConnection: conn, + driver, + pureName: collectionName, + createIfNotExists: true, + }); + await copyStream(importReader, importWriter); + + // Export + const exportReader = await tableReader({ + systemConnection: conn, + driver, + pureName: collectionName, + }); + const exportWriter = createExportStream(); + await copyStream(exportReader, exportWriter); + + const rows = exportWriter.resultArray.filter(x => !x.__isStreamHeader); + expect(rows.length).toBe(4); + + const sortedRows = rows.sort((a, b) => a._id.localeCompare(b._id)); + for (const doc of documents) { + const found = sortedRows.find(r => r._id === doc._id); + expect(found).toBeDefined(); + expect(found.name).toBe(doc.name); + expect(found.value).toBe(doc.value); + } + }); + + test('import documents with nested objects', async () => { + const documents = [ + { _id: 'nest1', name: 'Alice', address: { city: 'Prague', zip: '11000' } }, + { _id: 'nest2', name: 'Bob', address: { city: 'Vienna', zip: '1010' } }, + ]; + + const reader = createDocumentImportStream(documents); + const writer = await tableWriter({ + systemConnection: conn, + driver, + pureName: collectionName, + createIfNotExists: true, + }); + await copyStream(reader, writer); + + const all = await readAll(driver, conn, collectionName); + expect(all.rows.length).toBe(2); + + const alice = all.rows.find(r => r.name === 'Alice'); + expect(alice.address.city).toBe('Prague'); + expect(alice.address.zip).toBe('11000'); + }); + + test('import many documents', async () => { + const documents = []; + for (let i = 0; i < 150; i++) { + documents.push({ _id: `many${i}`, name: `Name${i}`, index: i }); + } + + const reader = createDocumentImportStream(documents); + const writer = await tableWriter({ + systemConnection: conn, + driver, + pureName: collectionName, + createIfNotExists: true, + }); + await copyStream(reader, writer); + + const result = await driver.readCollection(conn, { + pureName: collectionName, + countDocuments: true, + }); + expect(result.count).toBe(150); + }); + + test('export empty collection returns no data rows', async () => { + const reader = await tableReader({ + systemConnection: conn, + driver, + pureName: collectionName, + }); + const writer = createExportStream(); + await copyStream(reader, writer); + + const rows = writer.resultArray.filter(x => !x.__isStreamHeader); + expect(rows.length).toBe(0); + }); + }); +}); diff --git a/integration-tests/docker-compose.yaml b/integration-tests/docker-compose.yaml index e48b5ddb1..fd3dea5dd 100644 --- a/integration-tests/docker-compose.yaml +++ b/integration-tests/docker-compose.yaml @@ -123,5 +123,22 @@ services: retries: 3 start_period: 40s + mongodb: + image: mongo:4.0.12 + restart: always + volumes: + - mongo-data:/data/db + - mongo-config:/data/configdb + ports: + - 27017:27017 + + dynamodb: + image: amazon/dynamodb-local + restart: always + ports: + - 8000:8000 + volumes: firebird-data: + mongo-data: + mongo-config: diff --git a/integration-tests/engines.js b/integration-tests/engines.js index 528925e2c..45de0f317 100644 --- a/integration-tests/engines.js +++ b/integration-tests/engines.js @@ -738,6 +738,27 @@ const firebirdEngine = { skipDropReferences: true, }; +/** @type {import('dbgate-types').TestEngineInfo} */ +const mongoDbEngine = { + label: 'MongoDB', + connection: { + engine: 'mongo@dbgate-plugin-mongo', + server: 'localhost', + port: 27017, + }, +}; + +/** @type {import('dbgate-types').TestEngineInfo} */ +const dynamoDbEngine = { + label: 'DynamoDB', + connection: { + engine: 'dynamodb@dbgate-plugin-dynamodb', + server: 'localhost', + port: 8000, + authType: 'onpremise', + }, +}; + const enginesOnCi = [ // all engines, which would be run on GitHub actions mysqlEngine, @@ -788,3 +809,5 @@ module.exports.libsqlFileEngine = libsqlFileEngine; module.exports.libsqlWsEngine = libsqlWsEngine; module.exports.duckdbEngine = duckdbEngine; module.exports.firebirdEngine = firebirdEngine; +module.exports.mongoDbEngine = mongoDbEngine; +module.exports.dynamoDbEngine = dynamoDbEngine; diff --git a/integration-tests/wait.js b/integration-tests/wait.js index 2bdad2c6c..f5bfa9442 100644 --- a/integration-tests/wait.js +++ b/integration-tests/wait.js @@ -1,5 +1,6 @@ const requireEngineDriver = require('dbgate-api/src/utility/requireEngineDriver'); const engines = require('./engines'); +const { mongoDbEngine, dynamoDbEngine } = require('./engines'); global.DBGATE_PACKAGES = { 'dbgate-tools': require('dbgate-tools'), 'dbgate-sqltree': require('dbgate-sqltree'), @@ -9,7 +10,7 @@ global.DBGATE_PACKAGES = { async function connectEngine(engine) { const { connection } = engine; const driver = requireEngineDriver(connection); - for (;;) { + for (; ;) { try { const conn = await driver.connect(connection); await driver.getVersion(conn); @@ -26,7 +27,8 @@ async function connectEngine(engine) { async function run() { await new Promise(resolve => setTimeout(resolve, 10000)); - await Promise.all(engines.map(engine => connectEngine(engine))); + const documentEngines = [mongoDbEngine, dynamoDbEngine]; + await Promise.all([...engines, ...documentEngines].map(engine => connectEngine(engine))); } run(); diff --git a/workflow-templates/run-tests.yaml b/workflow-templates/run-tests.yaml index 720880381..33c7c00d4 100644 --- a/workflow-templates/run-tests.yaml +++ b/workflow-templates/run-tests.yaml @@ -138,3 +138,18 @@ jobs: FIREBIRD_USE_LEGACY_AUTH: true ports: - '3050:3050' + + mongodb: + image: mongo:4.0.12 + ports: + - '27017:27017' + restart: always + volumes: + - mongo-data:/data/db + - mongo-config:/data/configdb + + dynamodb: + image: amazon/dynamodb-local + ports: + - '8000:8000' + restart: always