Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
it("init", async function() {
this.timeout(30000);
// create temp directory
const projectDir = "df/examples/init";
// Project has already been initialized via the tests script, check data is valid.
// compile project
const graph = await compile({ projectDir }).catch(error => error);
expect(graph).to.not.be.an.instanceof(Error);
const gErrors = utils.validate(graph);
expect(gErrors)
.to.have.property("compilationErrors")
.to.be.an("array").that.is.empty;
expect(gErrors)
.to.have.property("validationErrors")
.to.be.an("array").that.is.empty;
});
});
export async function compile(
compileConfig: dataform.ICompileConfig
): Promise {
// Resolve the path in case it hasn't been resolved already.
path.resolve(compileConfig.projectDir);
const returnedPath = await CompileChildProcess.forkProcess().compile(compileConfig);
const contents = fs.readFileSync(returnedPath);
let compiledGraph = dataform.CompiledGraph.decode(contents);
fs.unlinkSync(returnedPath);
// Merge graph errors into the compiled graph.
compiledGraph = dataform.CompiledGraph.create({
...compiledGraph,
graphErrors: validate(compiledGraph)
});
return compiledGraph;
}
it("circular_dependencies", () => {
const session = new Session(path.dirname(__filename), TEST_CONFIG);
session.publish("a").dependencies("b");
session.publish("b").dependencies("a");
const cGraph = session.compile();
const gErrors = utils.validate(cGraph);
expect(gErrors)
.to.have.property("compilationErrors")
.to.be.an("array").that.is.not.empty;
expect(
gErrors.compilationErrors.filter(item => item.message.match(/Circular dependency/))
).to.be.an("array").that.is.not.empty;
});
it("ref", () => {
const session = new Session(path.dirname(__filename), TEST_CONFIG);
session.publish("a", _ => "select 1 as test");
session.publish("b", ctx => `select * from ${ctx.ref("a")}`);
session.publish("c", ctx => `select * from ${ctx.ref(undefined)}`);
session.publish("d", ctx => `select * from ${ctx.ref({ schema: "schema", name: "a" })}`);
session
.publish("e", {
schema: "foo"
})
.query(_ => "select 1 as test");
session.publish("f", ctx => `select * from ${ctx.ref("e")}`);
const graph = session.compile();
const graphErrors = utils.validate(graph);
const tableNames = graph.tables.map(item => item.name);
expect(tableNames).includes("schema.a");
expect(tableNames).includes("schema.b");
expect(tableNames).includes("schema.c");
expect(tableNames).includes("schema.d");
expect(tableNames).includes("foo.e");
expect(tableNames).includes("schema.f");
const errors = graphErrors.compilationErrors.map(item => item.message);
expect(errors).includes("Action name is not specified");
expect(graphErrors.compilationErrors.length === 1);
expect(graphErrors)
.to.have.property("validationErrors")
.to.be.an("array").that.is.empty;
});
it("same action names in different schemas", () => {
const session = new Session(path.dirname(__filename), TEST_CONFIG);
session.publish("b");
session.publish("a", { schema: "schema1" }).dependencies("b");
session.publish("a", { schema: "schema2" });
const cGraph = session.compile();
const gErrors = utils.validate(cGraph);
expect(gErrors)
.to.have.property("compilationErrors")
.to.be.an("array").that.is.empty;
});
});
it("snowflake compiles", async () => {
const graph = await compile({
projectDir: "df/examples/common_v1",
projectConfigOverride: { warehouse: "snowflake" }
}).catch(error => error);
expect(graph).to.not.be.an.instanceof(Error);
const gErrors = utils.validate(graph);
expect(gErrors)
.to.have.property("compilationErrors")
.to.be.an("array").that.is.empty;
expect(gErrors)
.to.have.property("validationErrors")
.to.be.an("array").that.is.empty;
const mNames = graph.tables.map((t: dataform.ITable) => t.name);
expect(mNames).includes("DF_INTEGRATION_TEST.EXAMPLE_INCREMENTAL");
const mIncremental = graph.tables.filter(
(t: dataform.ITable) => t.name === "DF_INTEGRATION_TEST.EXAMPLE_INCREMENTAL"
)[0];
expect(mIncremental.type).equals("incremental");
expect(mIncremental.query).equals(
it("validation_type", () => {
const sessionSuccess = new Session(path.dirname(__filename), TEST_CONFIG);
sessionSuccess.publish("exampleSuccess1", { type: "table" });
sessionSuccess.publish("exampleSuccess2", { type: "view" });
sessionSuccess.publish("exampleSuccess3", { type: "incremental" }).where("test");
const cgSuccess = sessionSuccess.compile();
const cgSuccessErrors = utils.validate(cgSuccess);
expect(cgSuccessErrors)
.to.have.property("validationErrors")
.to.be.an("array").that.is.empty;
const sessionFail = new Session(path.dirname(__filename), TEST_CONFIG);
sessionFail.publish("exampleFail", JSON.parse('{"type": "ta ble"}'));
const cgFail = sessionFail.compile();
const cgFailErrors = utils.validate(cgFail);
expect(cgFailErrors)
.to.have.property("validationErrors")
.to.be.an("array").that.is.not.empty;
const err = cgFailErrors.validationErrors.find(e => e.actionName === "schema.exampleFail");
expect(err)
.to.have.property("message")
.that.matches(/Wrong type of table/);
});
redshift: {}
});
const expectedResults = [
{ name: "schema.example_absent_distKey", message: /Property "distKey" is not defined/ },
{ name: "schema.example_absent_distStyle", message: /Property "distStyle" is not defined/ },
{ name: "schema.example_wrong_distStyle", message: /Wrong value of "distStyle" property/ },
{ name: "schema.example_absent_sortKeys", message: /Property "sortKeys" is not defined/ },
{ name: "schema.example_empty_sortKeys", message: /Property "sortKeys" is not defined/ },
{ name: "schema.example_absent_sortStyle", message: /Property "sortStyle" is not defined/ },
{ name: "schema.example_wrong_sortStyle", message: /Wrong value of "sortStyle" property/ },
{ name: "schema.example_empty_redshift", message: /Missing properties in redshift config/ }
];
const graph = session.compile();
const gErrors = utils.validate(graph);
expect(gErrors)
.to.have.property("validationErrors")
.to.be.an("array")
.to.have.lengthOf(8);
expectedResults.forEach(result => {
const err = gErrors.validationErrors.find(e => e.actionName === result.name);
expect(err)
.to.have.property("message")
.that.matches(result.message);
});
});
})
.preOps(_ => ["pre_op_b"])
.postOps(_ => ["post_op_b"])
.where("test_where")
.query(ctx => `select * from ${ctx.ref("a")}`);
session
.publish("c", {
type: "table",
columns: { test: "test description c" }
})
.preOps(_ => ["pre_op_c"])
.postOps(_ => ["post_op_c"])
.query(ctx => `select * from ${ctx.ref("b")}`);
const graph = session.compile();
const graphErrors = utils.validate(graph);
const tableA = graph.tables.find(item => item.name === "schema.a");
expect(tableA).to.exist;
expect(tableA.type).equals("table");
expect(tableA.dependencies).to.be.an("array").that.is.empty;
expect(tableA.query).equals("select 1 as test");
const tableB = graph.tables.find(item => item.name === "schema.b");
expect(tableB).to.exist;
expect(tableB.type).equals("inline");
expect(tableB.dependencies).includes("schema.a");
expect(tableB.actionDescriptor).eql({
columns: [
dataform.ColumnDescriptor.create({
description: "test description b",
path: ["test"]
const projectConfig = JSON.parse(dataformJson);
checkDataformJsonValidity({
...projectConfig,
...compileConfig.projectConfigOverride
});
} catch (e) {
throw new Error(`Compile Error: ProjectConfig ('dataform.json') is invalid. ${e}`);
}
const encodedGraphInBase64 = await CompileChildProcess.forkProcess().compile(compileConfig);
const encodedGraphBytes = new Uint8Array(util.base64.length(encodedGraphInBase64));
util.base64.decode(encodedGraphInBase64, encodedGraphBytes, 0);
const compiledGraph = dataform.CompiledGraph.decode(encodedGraphBytes);
return dataform.CompiledGraph.create({
...compiledGraph,
graphErrors: validate(compiledGraph)
});
}