From 108c145bdd8c7f6924e0c41adb634eae90baf9c9 Mon Sep 17 00:00:00 2001 From: Manlio Perillo Date: Thu, 20 Apr 2023 18:45:09 +0200 Subject: [PATCH 1/6] test: refactorize the code Currently, if there is an error when creating the patches/healed directory, the error message will be printed on stderr, but the build runner will report the test as being successful. Add the fail function and the FailStep, so that the error will be correctly handled by the build runner. Remove the PatchStep, and instead add the heal function so that all the exercises are healed before starting the tests. The heal function executes at the configuration phase, but the possible error is handled by the build runner. --- test/tests.zig | 80 +++++++++++++++++++++++++++++++------------------- 1 file changed, 49 insertions(+), 31 deletions(-) diff --git a/test/tests.zig b/test/tests.zig index c8f4af2..5bd1e82 100644 --- a/test/tests.zig +++ b/test/tests.zig @@ -2,8 +2,10 @@ const std = @import("std"); const root = @import("../build.zig"); const debug = std.debug; +const fmt = std.fmt; const fs = std.fs; +const Allocator = std.mem.Allocator; const Build = std.build; const Step = Build.Step; const RunStep = std.Build.RunStep; @@ -18,9 +20,10 @@ pub fn addCliTests(b: *std.Build, exercises: []const Exercise) *Step { const outdir = "patches/healed"; fs.cwd().makePath(outdir) catch |err| { - debug.print("unable to make '{s}': {s}\n", .{ outdir, @errorName(err) }); - - return step; + return fail(step, "unable to make '{s}': {s}\n", .{ outdir, @errorName(err) }); + }; + heal(b.allocator, exercises, outdir) catch |err| { + return fail(step, "unable to heal exercises: {s}\n", .{@errorName(err)}); }; { @@ -32,14 +35,11 @@ pub fn addCliTests(b: *std.Build, exercises: []const Exercise) *Step { i += 1; if (ex.skip) continue; - const patch = PatchStep.create(b, ex, outdir); - const cmd = b.addSystemCommand( &.{ b.zig_exe, "build", b.fmt("-Dn={}", .{i}), "-Dhealed", "test" }, ); cmd.setName(b.fmt("zig build -D={} -Dhealed test", .{i})); cmd.expectExitCode(0); - cmd.step.dependOn(&patch.step); // Some exercise output has an extra space character. if (ex.check_stdout) @@ -93,28 +93,21 @@ fn createCase(b: *Build, name: []const u8) *Step { return case_step; } -// Apply a patch to the specified exercise. -const PatchStep = struct { - const join = fs.path.join; - - const exercises_path = "exercises"; - const patches_path = "patches/patches"; - +// A step that will fail. +const FailStep = struct { step: Step, - exercise: Exercise, - outdir: []const u8, + error_msg: []const u8, - pub fn create(owner: *Build, exercise: Exercise, outdir: []const u8) *PatchStep { - const self = owner.allocator.create(PatchStep) catch @panic("OOM"); + pub fn create(owner: *Build, error_msg: []const u8) *FailStep { + const self = owner.allocator.create(FailStep) catch @panic("OOM"); self.* = .{ .step = Step.init(.{ .id = .custom, - .name = owner.fmt("patch {s}", .{exercise.main_file}), + .name = "fail", .owner = owner, .makeFn = make, }), - .exercise = exercise, - .outdir = outdir, + .error_msg = error_msg, }; return self; @@ -122,25 +115,50 @@ const PatchStep = struct { fn make(step: *Step, _: *std.Progress.Node) !void { const b = step.owner; - const self = @fieldParentPtr(PatchStep, "step", step); - const exercise = self.exercise; - const name = exercise.baseName(); + const self = @fieldParentPtr(FailStep, "step", step); + + try step.result_error_msgs.append(b.allocator, self.error_msg); + return error.MakeFailed; + } +}; + +// A variant of `std.Build.Step.fail` that does not return an error so that it +// can be used in the configuration phase. It returns a FailStep, so that the +// error will be cleanly handled by the build runner. +fn fail(step: *Step, comptime format: []const u8, args: anytype) *Step { + const b = step.owner; + + const fail_step = FailStep.create(b, b.fmt(format, args)); + step.dependOn(&fail_step.step); + + return step; +} + +// Heals all the exercises. +fn heal(allocator: Allocator, exercises: []const Exercise, outdir: []const u8) !void { + const join = fs.path.join; + + const exercises_path = "exercises"; + const patches_path = "patches/patches"; + + for (exercises) |ex| { + const name = ex.baseName(); // Use the POSIX patch variant. - const file = join(b.allocator, &.{ exercises_path, exercise.main_file }) catch - @panic("OOM"); - const patch = join(b.allocator, &.{ patches_path, b.fmt("{s}.patch", .{name}) }) catch - @panic("OOM"); - const output = join(b.allocator, &.{ self.outdir, exercise.main_file }) catch - @panic("OOM"); + const file = try join(allocator, &.{ exercises_path, ex.main_file }); + const patch = b: { + const patch_name = try fmt.allocPrint(allocator, "{s}.patch", .{name}); + break :b try join(allocator, &.{ patches_path, patch_name }); + }; + const output = try join(allocator, &.{ outdir, ex.main_file }); const argv = &.{ "patch", "-i", patch, "-o", output, file }; - var child = std.process.Child.init(argv, b.allocator); + var child = std.process.Child.init(argv, allocator); child.stdout_behavior = .Ignore; // the POSIX standard says that stdout is not used _ = try child.spawnAndWait(); } -}; +} // // Missing functions from std.Build.RunStep From cec0aa51dbb9bff30f871d3f605bc1f0fbc6b4a4 Mon Sep 17 00:00:00 2001 From: Manlio Perillo Date: Thu, 20 Apr 2023 21:13:13 +0200 Subject: [PATCH 2/6] test: fix incorrect cleanup code The current cleanup code is incorrect, since it may delete the healed directory while one test case is running. The solution is to make each test case isolate, with its own setup and teardown. Unfortunately it is currently not possible, since each test case modify the same directory. Disable the cleanup step, until a better solution is found. --- test/tests.zig | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/test/tests.zig b/test/tests.zig index 5bd1e82..015b55a 100644 --- a/test/tests.zig +++ b/test/tests.zig @@ -7,8 +7,8 @@ const fs = std.fs; const Allocator = std.mem.Allocator; const Build = std.build; -const Step = Build.Step; const RunStep = std.Build.RunStep; +const Step = Build.Step; const Exercise = root.Exercise; @@ -27,9 +27,9 @@ pub fn addCliTests(b: *std.Build, exercises: []const Exercise) *Step { }; { + // Test that `zig build -Dn=n -Dhealed test` selects the nth exercise. const case_step = createCase(b, "case-1"); - // Test that `zig build -Dn=n -Dhealed test` selects the nth exercise. var i: usize = 0; for (exercises[0 .. exercises.len - 1]) |ex| { i += 1; @@ -54,9 +54,9 @@ pub fn addCliTests(b: *std.Build, exercises: []const Exercise) *Step { } { + // Test that `zig build -Dn=n -Dhealed test` skips disabled esercises. const case_step = createCase(b, "case-2"); - // Test that `zig build -Dn=n -Dhealed test` skips disabled esercises. var i: usize = 0; for (exercises[0 .. exercises.len - 1]) |ex| { i += 1; @@ -76,8 +76,10 @@ pub fn addCliTests(b: *std.Build, exercises: []const Exercise) *Step { step.dependOn(case_step); } - const cleanup = b.addRemoveDirTree(outdir); - step.dependOn(&cleanup.step); + // Don't add the cleanup step, since it may delete outdir while a test case + // is running. + //const cleanup = b.addRemoveDirTree(outdir); + //step.dependOn(&cleanup.step); return step; } From 3dd7cdcb3b541b0da4f659aa15ea6d651a45f41a Mon Sep 17 00:00:00 2001 From: Manlio Perillo Date: Mon, 24 Apr 2023 11:20:07 +0200 Subject: [PATCH 3/6] build: make exercise output more reliable Currently, ZiglingStep prints the raw exercise output. This is not a problem when executing `zig build` from the shell, but in a unit test it is necessary to know when the exercise output ends. Document that Exercise.output should not have trailing whitespace. Ensure this is true by adding a check in the validate_exercises function. Remove trailing whitespace in exercises 68 and 99. Simplify the output validation in ZiglingStep.makeInternal. Checking that the length of the actual and expected output is the same is not necessary, since trailing whitespace has been removed. Simply do an exact comparison. Print the trimmed exercise output, instead of the raw output. This will ensure that the exercise output always ends with only one LF character. Fix some small coding style issues. --- build.zig | 46 ++++++++++++++++++++++++++++++---------------- 1 file changed, 30 insertions(+), 16 deletions(-) diff --git a/build.zig b/build.zig index 41cc64d..398f02e 100644 --- a/build.zig +++ b/build.zig @@ -18,7 +18,8 @@ pub const Exercise = struct { main_file: []const u8, /// This is the desired output of the program. - /// A program passes if its output ends with this string. + /// A program passes if its output, excluding trailing whitespace, is equal + /// to this string. output: []const u8, /// This is an optional hint to give if the program does not succeed. @@ -365,7 +366,7 @@ const exercises = [_]Exercise{ }, .{ .main_file = "068_comptime3.zig", - .output = "Minnow (1:32, 4 x 2)\nShark (1:16, 8 x 5)\nWhale (1:1, 143 x 95)\n", + .output = "Minnow (1:32, 4 x 2)\nShark (1:16, 8 x 5)\nWhale (1:1, 143 x 95)", }, .{ .main_file = "069_comptime4.zig", @@ -516,7 +517,7 @@ const exercises = [_]Exercise{ }, .{ .main_file = "099_formatting.zig", - .output = "\n X | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 \n---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+\n 1 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 \n\n 2 | 2 4 6 8 10 12 14 16 18 20 22 24 26 28 30 \n\n 3 | 3 6 9 12 15 18 21 24 27 30 33 36 39 42 45 \n\n 4 | 4 8 12 16 20 24 28 32 36 40 44 48 52 56 60 \n\n 5 | 5 10 15 20 25 30 35 40 45 50 55 60 65 70 75 \n\n 6 | 6 12 18 24 30 36 42 48 54 60 66 72 78 84 90 \n\n 7 | 7 14 21 28 35 42 49 56 63 70 77 84 91 98 105 \n\n 8 | 8 16 24 32 40 48 56 64 72 80 88 96 104 112 120 \n\n 9 | 9 18 27 36 45 54 63 72 81 90 99 108 117 126 135 \n\n10 | 10 20 30 40 50 60 70 80 90 100 110 120 130 140 150 \n\n11 | 11 22 33 44 55 66 77 88 99 110 121 132 143 154 165 \n\n12 | 12 24 36 48 60 72 84 96 108 120 132 144 156 168 180 \n\n13 | 13 26 39 52 65 78 91 104 117 130 143 156 169 182 195 \n\n14 | 14 28 42 56 70 84 98 112 126 140 154 168 182 196 210 \n\n15 | 15 30 45 60 75 90 105 120 135 150 165 180 195 210 225 \n\n", + .output = "\n X | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 \n---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+\n 1 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 \n\n 2 | 2 4 6 8 10 12 14 16 18 20 22 24 26 28 30 \n\n 3 | 3 6 9 12 15 18 21 24 27 30 33 36 39 42 45 \n\n 4 | 4 8 12 16 20 24 28 32 36 40 44 48 52 56 60 \n\n 5 | 5 10 15 20 25 30 35 40 45 50 55 60 65 70 75 \n\n 6 | 6 12 18 24 30 36 42 48 54 60 66 72 78 84 90 \n\n 7 | 7 14 21 28 35 42 49 56 63 70 77 84 91 98 105 \n\n 8 | 8 16 24 32 40 48 56 64 72 80 88 96 104 112 120 \n\n 9 | 9 18 27 36 45 54 63 72 81 90 99 108 117 126 135 \n\n10 | 10 20 30 40 50 60 70 80 90 100 110 120 130 140 150 \n\n11 | 11 22 33 44 55 66 77 88 99 110 121 132 143 154 165 \n\n12 | 12 24 36 48 60 72 84 96 108 120 132 144 156 168 180 \n\n13 | 13 26 39 52 65 78 91 104 117 130 143 156 169 182 195 \n\n14 | 14 28 42 56 70 84 98 112 126 140 154 168 182 196 210 \n\n15 | 15 30 45 60 75 90 105 120 135 150 165 180 195 210 225", }, .{ .main_file = "999_the_end.zig", @@ -758,20 +759,20 @@ const ZiglingStep = struct { return err; }; - // Allow up to 1 MB of stdout capture + // Allow up to 1 MB of stdout capture. const max_output_len = 1 * 1024 * 1024; const output = if (self.exercise.check_stdout) try child.stdout.?.reader().readAllAlloc(self.builder.allocator, max_output_len) else try child.stderr.?.reader().readAllAlloc(self.builder.allocator, max_output_len); - // at this point stdout is closed, wait for the process to terminate + // At this point stdout is closed, wait for the process to terminate. const term = child.wait() catch |err| { print("{s}Unable to spawn {s}: {s}{s}\n", .{ red_text, argv[0], @errorName(err), reset_text }); return err; }; - // make sure it exited cleanly. + // Make sure it exited cleanly. switch (term) { .Exited => |code| { if (code != 0) { @@ -785,10 +786,10 @@ const ZiglingStep = struct { }, } - // validate the output + // Validate the output. const trimOutput = std.mem.trimRight(u8, output, " \r\n"); const trimExerciseOutput = std.mem.trimRight(u8, self.exercise.output, " \r\n"); - if (std.mem.indexOf(u8, trimOutput, trimExerciseOutput) == null or trimOutput.len != trimExerciseOutput.len) { + if (!std.mem.eql(u8, trimOutput, trimExerciseOutput)) { print( \\ \\{s}----------- Expected this output -----------{s} @@ -801,7 +802,7 @@ const ZiglingStep = struct { return error.InvalidOutput; } - print("{s}PASSED:\n{s}{s}\n", .{ green_text, output, reset_text }); + print("{s}PASSED:\n{s}{s}\n\n", .{ green_text, trimOutput, reset_text }); } // The normal compile step calls os.exit, so we can't use it as a library :( @@ -1057,18 +1058,31 @@ const SkipStep = struct { } }; -// Check that each exercise number, excluding the last, forms the sequence `[1, exercise.len)`. +// Check that each exercise number, excluding the last, forms the sequence +// `[1, exercise.len)`. +// +// Additionally check that the output field does not contain trailing whitespace. fn validate_exercises() bool { - // Don't use the "multi-object for loop" syntax, in order to avoid a syntax error with old Zig - // compilers. + // Don't use the "multi-object for loop" syntax, in order to avoid a syntax + // error with old Zig compilers. var i: usize = 0; for (exercises[0 .. exercises.len - 1]) |ex| { i += 1; if (ex.number() != i) { - print( - "exercise {s} has an incorrect number: expected {}, got {s}\n", - .{ ex.main_file, i, ex.key() }, - ); + print("exercise {s} has an incorrect number: expected {}, got {s}\n", .{ + ex.main_file, + i, + ex.key(), + }); + + return false; + } + + const output = std.mem.trimRight(u8, ex.output, " \r\n"); + if (output.len != ex.output.len) { + print("exercise {s} output field has extra trailing whitespace\n", .{ + ex.main_file, + }); return false; } From 131772edd7f475b3b2331117ed1d833ab926a156 Mon Sep 17 00:00:00 2001 From: Manlio Perillo Date: Sat, 22 Apr 2023 17:22:34 +0200 Subject: [PATCH 4/6] test: add test for `zig build` and `zig build -Dn=1 start` Add tests for `zig build` and `zig build -Dn=1 start`, in order to test that the all the exercises are processed in the correct order. --- test/tests.zig | 166 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 166 insertions(+) diff --git a/test/tests.zig b/test/tests.zig index 015b55a..ee38f49 100644 --- a/test/tests.zig +++ b/test/tests.zig @@ -4,9 +4,12 @@ const root = @import("../build.zig"); const debug = std.debug; const fmt = std.fmt; const fs = std.fs; +const mem = std.mem; const Allocator = std.mem.Allocator; const Build = std.build; +const FileSource = std.Build.FileSource; +const Reader = fs.File.Reader; const RunStep = std.Build.RunStep; const Step = Build.Step; @@ -76,6 +79,45 @@ pub fn addCliTests(b: *std.Build, exercises: []const Exercise) *Step { step.dependOn(case_step); } + { + // Test that `zig build -Dhealed` process all the exercises in order. + const case_step = createCase(b, "case-3"); + + // TODO: when an exercise is modified, the cache is not invalidated. + const cmd = b.addSystemCommand(&.{ b.zig_exe, "build", "-Dhealed" }); + cmd.setName("zig build -Dhealed"); + cmd.expectExitCode(0); + + const stderr = cmd.captureStdErr(); + const verify = CheckStep.create(b, exercises, stderr, true); + verify.step.dependOn(&cmd.step); + + case_step.dependOn(&verify.step); + + step.dependOn(case_step); + } + + { + // Test that `zig build -Dhealed -Dn=1 start` process all the exercises + // in order. + const case_step = createCase(b, "case-4"); + + // TODO: when an exercise is modified, the cache is not invalidated. + const cmd = b.addSystemCommand( + &.{ b.zig_exe, "build", "-Dhealed", "-Dn=1", "start" }, + ); + cmd.setName("zig build -Dhealed -Dn=1 start"); + cmd.expectExitCode(0); + + const stderr = cmd.captureStdErr(); + const verify = CheckStep.create(b, exercises, stderr, false); + verify.step.dependOn(&cmd.step); + + case_step.dependOn(&verify.step); + + step.dependOn(case_step); + } + // Don't add the cleanup step, since it may delete outdir while a test case // is running. //const cleanup = b.addRemoveDirTree(outdir); @@ -95,6 +137,130 @@ fn createCase(b: *Build, name: []const u8) *Step { return case_step; } +// Check the output of `zig build` or `zig build -Dn=1 start`. +const CheckStep = struct { + step: Step, + exercises: []const Exercise, + stderr: FileSource, + has_logo: bool, + + pub fn create( + owner: *Build, + exercises: []const Exercise, + stderr: FileSource, + has_logo: bool, + ) *CheckStep { + const self = owner.allocator.create(CheckStep) catch @panic("OOM"); + self.* = .{ + .step = Step.init(.{ + .id = .custom, + .name = "check", + .owner = owner, + .makeFn = make, + }), + .exercises = exercises, + .stderr = stderr, + .has_logo = has_logo, + }; + + return self; + } + + fn make(step: *Step, _: *std.Progress.Node) !void { + const b = step.owner; + const self = @fieldParentPtr(CheckStep, "step", step); + const exercises = self.exercises; + + const stderr_file = try fs.cwd().openFile( + self.stderr.getPath(b), + .{ .mode = .read_only }, + ); + defer stderr_file.close(); + + const stderr = stderr_file.reader(); + for (exercises) |ex| { + if (ex.number() == 1 and self.has_logo) { + // Skip the logo. + var buf: [80]u8 = undefined; + + var lineno: usize = 0; + while (lineno < 8) : (lineno += 1) { + _ = try readLine(stderr, &buf); + } + } + try check_output(step, ex, stderr); + } + } + + fn check_output(step: *Step, exercise: Exercise, reader: Reader) !void { + const b = step.owner; + + var buf: [1024]u8 = undefined; + if (exercise.skip) { + { + const actual = try readLine(reader, &buf) orelse "EOF"; + const expect = b.fmt("Skipping {s}", .{exercise.main_file}); + try check(step, exercise, expect, actual); + } + + { + const actual = try readLine(reader, &buf) orelse "EOF"; + try check(step, exercise, "", actual); + } + + return; + } + + { + const actual = try readLine(reader, &buf) orelse "EOF"; + const expect = b.fmt("Compiling {s}...", .{exercise.main_file}); + try check(step, exercise, expect, actual); + } + + { + const actual = try readLine(reader, &buf) orelse "EOF"; + const expect = b.fmt("Checking {s}...", .{exercise.main_file}); + try check(step, exercise, expect, actual); + } + + { + const actual = try readLine(reader, &buf) orelse "EOF"; + const expect = "PASSED:"; + try check(step, exercise, expect, actual); + } + + // Skip the exercise output. + const nlines = 1 + mem.count(u8, exercise.output, "\n") + 1; + var lineno: usize = 0; + while (lineno < nlines) : (lineno += 1) { + _ = try readLine(reader, &buf) orelse @panic("EOF"); + } + } + + fn check( + step: *Step, + exercise: Exercise, + expect: []const u8, + actual: []const u8, + ) !void { + if (!mem.eql(u8, expect, actual)) { + return step.fail("{s}: expected to see \"{s}\", found \"{s}\"", .{ + exercise.main_file, + expect, + actual, + }); + } + } + + fn readLine(reader: fs.File.Reader, buf: []u8) !?[]const u8 { + if (try reader.readUntilDelimiterOrEof(buf, '\n')) |line| { + return mem.trimRight(u8, line, " \r\n"); + } + + return null; + } +}; + // A step that will fail. const FailStep = struct { step: Step, From 5b2e8421578a02379799415ace72acb85f573bcd Mon Sep 17 00:00:00 2001 From: Manlio Perillo Date: Mon, 24 Apr 2023 12:19:00 +0200 Subject: [PATCH 5/6] test: change the order of `zig build` options In test case 1 and 2, move the -Dhealed option before the -Dn option, for consistency. Fix a typo in cmd.setName in test case 1 and 2. Remove a confusing comment in test case 1. --- test/tests.zig | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/test/tests.zig b/test/tests.zig index ee38f49..069c3b6 100644 --- a/test/tests.zig +++ b/test/tests.zig @@ -30,7 +30,7 @@ pub fn addCliTests(b: *std.Build, exercises: []const Exercise) *Step { }; { - // Test that `zig build -Dn=n -Dhealed test` selects the nth exercise. + // Test that `zig build -Dhealed -Dn=n test` selects the nth exercise. const case_step = createCase(b, "case-1"); var i: usize = 0; @@ -39,12 +39,11 @@ pub fn addCliTests(b: *std.Build, exercises: []const Exercise) *Step { if (ex.skip) continue; const cmd = b.addSystemCommand( - &.{ b.zig_exe, "build", b.fmt("-Dn={}", .{i}), "-Dhealed", "test" }, + &.{ b.zig_exe, "build", "-Dhealed", b.fmt("-Dn={}", .{i}), "test" }, ); - cmd.setName(b.fmt("zig build -D={} -Dhealed test", .{i})); + cmd.setName(b.fmt("zig build -Dhealed -Dn={} test", .{i})); cmd.expectExitCode(0); - // Some exercise output has an extra space character. if (ex.check_stdout) expectStdOutMatch(cmd, ex.output) else @@ -57,7 +56,7 @@ pub fn addCliTests(b: *std.Build, exercises: []const Exercise) *Step { } { - // Test that `zig build -Dn=n -Dhealed test` skips disabled esercises. + // Test that `zig build -Dhealed -Dn=n test` skips disabled esercises. const case_step = createCase(b, "case-2"); var i: usize = 0; @@ -66,9 +65,9 @@ pub fn addCliTests(b: *std.Build, exercises: []const Exercise) *Step { if (!ex.skip) continue; const cmd = b.addSystemCommand( - &.{ b.zig_exe, "build", b.fmt("-Dn={}", .{i}), "-Dhealed", "test" }, + &.{ b.zig_exe, "build", "-Dhealed", b.fmt("-Dn={}", .{i}), "test" }, ); - cmd.setName(b.fmt("zig build -D={} -Dhealed test", .{i})); + cmd.setName(b.fmt("zig build -Dhealed -Dn={} test", .{i})); cmd.expectExitCode(0); cmd.expectStdOutEqual(""); expectStdErrMatch(cmd, b.fmt("{s} skipped", .{ex.main_file})); From 4f690b074a4a2eed4f7062515dfcccae6d1725c8 Mon Sep 17 00:00:00 2001 From: Manlio Perillo Date: Mon, 24 Apr 2023 13:03:52 +0200 Subject: [PATCH 6/6] test: add a test for `zig build -Dn=1` Add a test for `zig build -Dn=1` in order to test that a broken exercise will print an hint. --- test/tests.zig | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/test/tests.zig b/test/tests.zig index 069c3b6..f91c4fd 100644 --- a/test/tests.zig +++ b/test/tests.zig @@ -117,6 +117,20 @@ pub fn addCliTests(b: *std.Build, exercises: []const Exercise) *Step { step.dependOn(case_step); } + { + // Test that `zig build -Dn=1` prints the hint. + const case_step = createCase(b, "case-5"); + + const cmd = b.addSystemCommand(&.{ b.zig_exe, "build", "-Dn=1" }); + cmd.setName("zig build -Dn=1"); + cmd.expectExitCode(1); + expectStdErrMatch(cmd, exercises[0].hint); + + case_step.dependOn(&cmd.step); + + step.dependOn(case_step); + } + // Don't add the cleanup step, since it may delete outdir while a test case // is running. //const cleanup = b.addRemoveDirTree(outdir);