diff --git a/plugins/data-replication/index.ts b/plugins/data-replication/index.ts new file mode 100644 index 0000000..32d7447 --- /dev/null +++ b/plugins/data-replication/index.ts @@ -0,0 +1,30 @@ +export default { + async run(config: any) { + const { + EXTERNAL_DB_TYPE, + EXTERNAL_DB_HOST, + REPLICATION_INTERVAL = 60, // dalam detik + TABLES_TO_SYNC + } = config; + + console.log(`Starting replication from ${EXTERNAL_DB_HOST}...`); + + // Fungsi utama untuk narik data + const syncData = async () => { + for (const table of TABLES_TO_SYNC) { + try { + // 1. Cek ID terakhir di internal SQLite + // 2. Tarik data dari External DB yang ID > ID_Internal + // 3. Simpan ke SQLite internal + console.log(`Syncing table: ${table}`); + } catch (error) { + console.error(`Failed to sync ${table}:`, error); + } + } + }; + + // Menjalankan sinkronisasi berdasarkan interval + setInterval(syncData, REPLICATION_INTERVAL * 1000); + } +}; + diff --git a/plugins/data-replication/plugin.json b/plugins/data-replication/plugin.json new file mode 100644 index 0000000..9d54a89 --- /dev/null +++ b/plugins/data-replication/plugin.json @@ -0,0 +1,7 @@ +{ + "id": "data-replication-plugin", + "name": "External Data Replicator", + "description": "Replicate data from external sources (Postgres, etc) to local SQLite", + "version": "1.0.0" +} + diff --git a/src/export/dump.ts b/src/export/dump.ts index 91a2e89..acaa68e 100644 --- a/src/export/dump.ts +++ b/src/export/dump.ts @@ -8,64 +8,73 @@ export async function dumpDatabaseRoute( config: StarbaseDBConfiguration ): Promise { try { - // Get all table names - const tablesResult = await executeOperation( - [{ sql: "SELECT name FROM sqlite_master WHERE type='table';" }], - dataSource, - config - ) + const { readable, writable } = new TransformStream(); + const writer = writable.getWriter(); + const encoder = new TextEncoder(); - const tables = tablesResult.map((row: any) => row.name) - let dumpContent = 'SQLite format 3\0' // SQLite file header - - // Iterate through all tables - for (const table of tables) { - // Get table schema - const schemaResult = await executeOperation( - [ - { - sql: `SELECT sql FROM sqlite_master WHERE type='table' AND name='${table}';`, - }, - ], - dataSource, - config - ) + // Proses streaming berjalan di background + (async () => { + try { + // 1. Ambil semua nama tabel + const tablesResult = await executeOperation( + [{ sql: "SELECT name FROM sqlite_master WHERE type='table';" }], + dataSource, + config + ) + const tables = tablesResult.map((row: any) => row.name) - if (schemaResult.length) { - const schema = schemaResult[0].sql - dumpContent += `\n-- Table: ${table}\n${schema};\n\n` - } + // 2. Kirim Header SQLite + await writer.write(encoder.encode('SQLite format 3\0')) - // Get table data - const dataResult = await executeOperation( - [{ sql: `SELECT * FROM ${table};` }], - dataSource, - config - ) + for (const table of tables) { + // 3. Ambil dan kirim Schema Tabel + const schemaResult = await executeOperation( + [{ sql: `SELECT sql FROM sqlite_master WHERE type='table' AND name='${table}';` }], + dataSource, + config + ) - for (const row of dataResult) { - const values = Object.values(row).map((value) => - typeof value === 'string' - ? `'${value.replace(/'/g, "''")}'` - : value - ) - dumpContent += `INSERT INTO ${table} VALUES (${values.join(', ')});\n` - } + if (schemaResult.length) { + const schema = schemaResult[0].sql + await writer.write(encoder.encode(`\n-- Table: ${table}\n${schema};\n\n`)) + } - dumpContent += '\n' - } + // 4. Ambil Data Tabel (Streaming per baris) + const dataResult = await executeOperation( + [{ sql: `SELECT * FROM ${table};` }], + dataSource, + config + ) - // Create a Blob from the dump content - const blob = new Blob([dumpContent], { type: 'application/x-sqlite3' }) + for (const row of dataResult) { + const values = Object.values(row).map((value) => + typeof value === 'string' + ? `'${value.replace(/'/g, "''")}'` + : value + ) + const line = `INSERT INTO ${table} VALUES (${values.join(', ')});\n` + await writer.write(encoder.encode(line)) + } + await writer.write(encoder.encode('\n')) + } + } catch (err) { + console.error("Streaming Error:", err) + } finally { + await writer.close() + } + })() - const headers = new Headers({ - 'Content-Type': 'application/x-sqlite3', - 'Content-Disposition': 'attachment; filename="database_dump.sql"', + // 5. Kembalikan Response berupa Stream (Gak pake Blob lagi!) + return new Response(readable, { + headers: { + 'Content-Type': 'application/x-sqlite3', + 'Content-Disposition': 'attachment; filename="database_dump.sql"', + 'Transfer-Encoding': 'chunked' + } }) - - return new Response(blob, { headers }) } catch (error: any) { console.error('Database Dump Error:', error) return createResponse(undefined, 'Failed to create database dump', 500) } } +