*N.B: This post duplicate the issue 1547 @ dataform-co/dataform * Hi,
We are migrating code to dataform and we need to:
- Make a call to bigquery API to get a list of columns
- Use list of column to replace within a SQL query template
- Execute the SQL query
Question
Is it possible to use such async logics where we need to interact with external async API ?
We tried two alternatives:
Attempt 1: Using promise in a js block within .sqlx file
Code
config {
type: "table",
disabled: false,
hasOutput: true,
description: "This table contains sales orders",
tags: ["dedup_test"]
}
js {
const common = require("our_custom_lib");
const deduplicationRequest = new common.deduplication.DeduplicationRequest(someRequestParameters...);
async function output() {
return await dedup.getDeduplicationRequest(deduplicationRequest)
}
}
${output().then(res => res)}
Output
Compiling...
Compiled successfully.
Running...
Operation failed: dataform.dedup_test
>
>
>
>
> [object Promise]
bigquery error: Syntax error
Attempt 2: Using promise in .js file
Code
const common = require("our_custom_lib");
const params = {
database: "our_database",
schema: "dataform",
type: "table",
disabled: false,
description: "This table contains sales orders",
tags: ["dedup_test"]
}
async function executeQuery(params) {
(async () => {
const deduplicationRequest = new common.deduplication.DeduplicationRequest(someRequestParameters...)
// Our async logic using bigquery JS client
return await common.deduplication.getDeltaDeduplicationQuery(deduplicationRequest)
}
).then( dedupQuery => {
// dataform-core
publish("transactions",params).query(dedupQuery);
}
)
}
executeQuery(params);
Output
Compilation runs fine but the table does not appear in compiled tables.