Skip to content

Commit d5e0ed8

Browse files
benjiebenjie
authored
Add ability to include external files in current.sql (#195)
2 parents dc62a35 + 062003f commit d5e0ed8

22 files changed

+398
-99
lines changed

‎README.md‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -724,6 +724,59 @@ by `graphile-migrate watch` is defined. By default this is in the
724724
`migrations/current.sql` file, but it might be `migrations/current/*.sql` if
725725
you're using folder mode.
726726
727+
#### Including external files in the current migration
728+
729+
You can include external files in your `current.sql` to better assist in source
730+
control. These includes are identified by paths within the `migrations/fixtures`
731+
folder.
732+
733+
For example. Given the following directory structure:
734+
735+
```
736+
/- migrate
737+
- migrations
738+
|
739+
- current.sql
740+
- fixtures
741+
|
742+
- functions
743+
|
744+
- myfunction.sql
745+
```
746+
747+
and the contents of `myfunction.sql`:
748+
749+
```sql
750+
create or replace function myfunction(a int, b int)
751+
returns int as $$
752+
select a + b;
753+
$$ language sql stable;
754+
```
755+
756+
When you make changes to `myfunction.sql`, include it in your current migration
757+
by adding `--!include functions/myfunction.sql` to your `current.sql` (or any
758+
`current/*.sql`). This statement doesn't need to be at the top of the file,
759+
wherever it is will be replaced by the content of
760+
`migrations/fixtures/functions/myfunction.sql` when the migration is committed.
761+
762+
```sql
763+
--!include fixtures/functions/myfunction.sql
764+
drop policy if exists access_by_numbers on mytable;
765+
create policy access_by_numbers on mytable for update using (myfunction(4, 2) < 42);
766+
```
767+
768+
and when the migration is committed or watched, the contents of `myfunction.sql`
769+
will be included in the result, such that the following SQL is executed:
770+
771+
```sql
772+
create or replace function myfunction(a int, b int)
773+
returns int as $$
774+
select a + b;
775+
$$ language sql stable;
776+
drop policy if exists access_by_numbers on mytable;
777+
create policy access_by_numbers on mytable for update using (myfunction(4, 2) < 42);
778+
```
779+
727780
### Committed migration(s)
728781
729782
The files for migrations that you've committed with `graphile-migrate commit`

‎__tests__/__snapshots__/include.test.ts.snap‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
// Jest Snapshot v1, https://goo.gl/fbAQLP
2+
3+
exports[`compiles an included file, and won't get stuck in an infinite include loop 1`] = `
4+
"Circular include detected - '~/migrations/fixtures/foo.sql' is included again! Import statement: \`--!include foo.sql\`; trace:
5+
~/migrations/fixtures/foo.sql
6+
~/migrations/current.sql"
7+
`;
8+
9+
exports[`disallows calling files outside of the migrations/fixtures folder 1`] = `"Forbidden: cannot include path '~/outsideFolder/foo.sql' because it's not inside '~/migrations/fixtures'"`;

‎__tests__/commit.test.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
import "./helpers"; // Has side-effects; must come first
22

3-
import { promises as fsp } from "fs";
3+
import * as fsp from "fs/promises";
44
import mockFs from "mock-fs";
55

66
import { commit } from "../src";

‎__tests__/compile.test.ts‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,8 @@
11
import "./helpers";
22

3-
import { compile } from "../src";
3+
import * as mockFs from "mock-fs";
44

5+
import { compile } from "../src";
56
let old: string | undefined;
67
beforeAll(() => {
78
old = process.env.DATABASE_AUTHENTICATOR;
@@ -11,6 +12,10 @@ afterAll(() => {
1112
process.env.DATABASE_AUTHENTICATOR = old;
1213
});
1314

15+
afterEach(() => {
16+
mockFs.restore();
17+
});
18+
1419
it("compiles SQL with settings", async () => {
1520
expect(
1621
await compile(

‎__tests__/include.test.ts‎

Lines changed: 145 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,145 @@
1+
import "./helpers";
2+
3+
import mockFs from "mock-fs";
4+
5+
import { compileIncludes } from "../src/migration";
6+
import { ParsedSettings, parseSettings } from "../src/settings";
7+
8+
let old: string | undefined;
9+
let settings: ParsedSettings;
10+
beforeAll(async () => {
11+
old = process.env.DATABASE_AUTHENTICATOR;
12+
process.env.DATABASE_AUTHENTICATOR = "dbauth";
13+
settings = await parseSettings({
14+
connectionString: "postgres://dbowner:dbpassword@dbhost:1221/dbname",
15+
placeholders: {
16+
":DATABASE_AUTHENTICATOR": "!ENV",
17+
},
18+
migrationsFolder: "migrations",
19+
});
20+
});
21+
afterAll(() => {
22+
process.env.DATABASE_AUTHENTICATOR = old;
23+
});
24+
25+
afterEach(() => {
26+
mockFs.restore();
27+
});
28+
29+
/** Pretents that our compiled files are 'current.sql' */
30+
const FAKE_VISITED = new Set([`${process.cwd()}/migrations/current.sql`]);
31+
32+
it("compiles an included file", async () => {
33+
mockFs({
34+
"migrations/fixtures/foo.sql": "select * from foo;",
35+
});
36+
expect(
37+
await compileIncludes(
38+
settings,
39+
`\
40+
--!include foo.sql
41+
`,
42+
FAKE_VISITED,
43+
),
44+
).toEqual(`\
45+
select * from foo;
46+
`);
47+
});
48+
49+
it("compiles multiple included files", async () => {
50+
mockFs({
51+
"migrations/fixtures/dir1/foo.sql": "select * from foo;",
52+
"migrations/fixtures/dir2/bar.sql": "select * from bar;",
53+
"migrations/fixtures/dir3/baz.sql": "--!include dir4/qux.sql",
54+
"migrations/fixtures/dir4/qux.sql": "select * from qux;",
55+
});
56+
expect(
57+
await compileIncludes(
58+
settings,
59+
`\
60+
--!include dir1/foo.sql
61+
--!include dir2/bar.sql
62+
--!include dir3/baz.sql
63+
`,
64+
FAKE_VISITED,
65+
),
66+
).toEqual(`\
67+
select * from foo;
68+
select * from bar;
69+
select * from qux;
70+
`);
71+
});
72+
73+
it("compiles an included file, and won't get stuck in an infinite include loop", async () => {
74+
mockFs({
75+
"migrations/fixtures/foo.sql": "select * from foo;\n--!include foo.sql",
76+
});
77+
const promise = compileIncludes(
78+
settings,
79+
`\
80+
--!include foo.sql
81+
`,
82+
FAKE_VISITED,
83+
);
84+
await expect(promise).rejects.toThrowError(/Circular include/);
85+
const message = await promise.catch((e) => e.message);
86+
expect(message.replaceAll(process.cwd(), "~")).toMatchSnapshot();
87+
});
88+
89+
it("disallows calling files outside of the migrations/fixtures folder", async () => {
90+
mockFs({
91+
"migrations/fixtures/bar.sql": "",
92+
"outsideFolder/foo.sql": "select * from foo;",
93+
});
94+
95+
const promise = compileIncludes(
96+
settings,
97+
`\
98+
--!include ../../outsideFolder/foo.sql
99+
`,
100+
FAKE_VISITED,
101+
);
102+
await expect(promise).rejects.toThrowError(/Forbidden: cannot include/);
103+
const message = await promise.catch((e) => e.message);
104+
expect(message.replaceAll(process.cwd(), "~")).toMatchSnapshot();
105+
});
106+
107+
it("compiles an included file that contains escapable things", async () => {
108+
mockFs({
109+
"migrations/fixtures/foo.sql": `\
110+
begin;
111+
112+
create or replace function current_user_id() returns uuid as $$
113+
select nullif(current_setting('user.id', true)::text, '')::uuid;
114+
$$ language sql stable;
115+
116+
comment on function current_user_id is E'The ID of the current user.';
117+
118+
grant all on function current_user_id to :DATABASE_USER;
119+
120+
commit;
121+
`,
122+
});
123+
expect(
124+
await compileIncludes(
125+
settings,
126+
`\
127+
--!include foo.sql
128+
`,
129+
FAKE_VISITED,
130+
),
131+
).toEqual(`\
132+
begin;
133+
134+
create or replace function current_user_id() returns uuid as $$
135+
select nullif(current_setting('user.id', true)::text, '')::uuid;
136+
$$ language sql stable;
137+
138+
comment on function current_user_id is E'The ID of the current user.';
139+
140+
grant all on function current_user_id to :DATABASE_USER;
141+
142+
commit;
143+
144+
`);
145+
});

‎__tests__/readCurrentMigration.test.ts‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -102,3 +102,14 @@ With multiple lines
102102
const content = await readCurrentMigration(parsedSettings, currentLocation);
103103
expect(content).toEqual(contentWithSplits);
104104
});
105+
106+
it("reads from current.sql, and processes included files", async () => {
107+
mockFs({
108+
"migrations/current.sql": "--!include foo_current.sql",
109+
"migrations/fixtures/foo_current.sql": "-- TEST from foo",
110+
});
111+
112+
const currentLocation = await getCurrentMigrationLocation(parsedSettings);
113+
const content = await readCurrentMigration(parsedSettings, currentLocation);
114+
expect(content).toEqual("-- TEST from foo");
115+
});

‎__tests__/uncommit.test.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
import "./helpers"; // Has side-effects; must come first
22

3-
import { promises as fsp } from "fs";
3+
import * as fsp from "fs/promises";
44
import mockFs from "mock-fs";
55

66
import { commit, migrate, uncommit } from "../src";

‎__tests__/writeCurrentMigration.test.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
import "./helpers"; // Has side-effects; must come first
22

3-
import { promises as fsp } from "fs";
3+
import * as fsp from "fs/promises";
44
import mockFs from "mock-fs";
55

66
import {

‎package.json‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -42,12 +42,12 @@
4242
"dependencies": {
4343
"@graphile/logger": "^0.2.0",
4444
"@types/json5": "^2.2.0",
45-
"@types/node": "^20.11.5",
46-
"@types/pg": "^8.10.9",
45+
"@types/node": "^18",
46+
"@types/pg": ">=6 <9",
4747
"chalk": "^4",
4848
"chokidar": "^3.5.3",
4949
"json5": "^2.2.3",
50-
"pg": "^8.11.3",
50+
"pg": ">=6.5 <9",
5151
"pg-connection-string": "^2.6.2",
5252
"pg-minify": "^1.6.3",
5353
"tslib": "^2.6.2",

‎scripts/update-docs.js‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
#!/usr/bin/env node
2-
const { promises: fsp } = require("fs");
2+
const fsp = require("fs/promises");
33
const { spawnSync } = require("child_process");
44

55
async function main() {

0 commit comments

Comments
 (0)