Refactor testing support

Following the implementation in `std.Build.Step.Compile, add the Kind
type to differentiate between a normal executable and a test executable
running zig tests.  Replace `Exercise.run_test` field with `kind`.

Compile the exercise in both the exe and test cases, reducing code
duplication.

Add the `check_output` and `check_test` methods in ZiglingStep, in order
to differentiate the code checking a normal executable and a test
executable.

Update the tests to correctly check both the exe and test cases.  Remove
the temporary code added in commit 832772c.
This commit is contained in:
Manlio Perillo 2023-05-15 15:23:10 +02:00
parent 3dafa3518b
commit 9ab9ebf33f
2 changed files with 58 additions and 85 deletions

115
build.zig
View file

@ -13,6 +13,13 @@ const assert = std.debug.assert;
const join = std.fs.path.join; const join = std.fs.path.join;
const print = std.debug.print; const print = std.debug.print;
const Kind = enum {
/// Run the artifact as a normal executable.
exe,
/// Run the artifact as a test.
@"test",
};
pub const Exercise = struct { pub const Exercise = struct {
/// main_file must have the format key_name.zig. /// main_file must have the format key_name.zig.
/// The key will be used as a shorthand to build just one example. /// The key will be used as a shorthand to build just one example.
@ -34,9 +41,8 @@ pub const Exercise = struct {
/// We need to keep track of this, so we compile with libc. /// We need to keep track of this, so we compile with libc.
link_libc: bool = false, link_libc: bool = false,
/// This exercise doesn't have a main function. /// This exercise kind.
/// We only call the test. kind: Kind = .exe,
run_test: bool = false,
/// This exercise is not supported by the current Zig compiler. /// This exercise is not supported by the current Zig compiler.
skip: bool = false, skip: bool = false,
@ -225,18 +231,6 @@ const ZiglingStep = struct {
return; return;
} }
// Test exercise.
if (self.exercise.run_test) {
self.is_testing = true;
const result_msg = self.testing(prog_node) catch {
std.os.exit(2);
};
const output = try trimLines(self.step.owner.allocator, result_msg);
print("\n{s}PASSED:\n{s}{s}\n\n", .{ green_text, output, reset_text });
return;
}
// Normal exercise.
const exe_path = self.compile(prog_node) catch { const exe_path = self.compile(prog_node) catch {
if (self.exercise.hint) |hint| if (self.exercise.hint) |hint|
print("\n{s}Ziglings hint: {s}{s}", .{ bold_text, hint, reset_text }); print("\n{s}Ziglings hint: {s}{s}", .{ bold_text, hint, reset_text });
@ -276,10 +270,14 @@ const ZiglingStep = struct {
return err; return err;
}; };
const raw_output = if (self.exercise.check_stdout) switch (self.exercise.kind) {
result.stdout .exe => return self.check_output(result),
else .@"test" => return self.check_test(result),
result.stderr; }
}
fn check_output(self: *ZiglingStep, result: Child.ExecResult) !void {
const b = self.step.owner;
// Make sure it exited cleanly. // Make sure it exited cleanly.
switch (result.term) { switch (result.term) {
@ -299,6 +297,11 @@ const ZiglingStep = struct {
}, },
} }
const raw_output = if (self.exercise.check_stdout)
result.stdout
else
result.stderr;
// Validate the output. // Validate the output.
// NOTE: exercise.output can never contain a CR character. // NOTE: exercise.output can never contain a CR character.
// See https://ziglang.org/documentation/master/#Source-Encoding. // See https://ziglang.org/documentation/master/#Source-Encoding.
@ -323,55 +326,28 @@ const ZiglingStep = struct {
print("{s}PASSED:\n{s}{s}\n\n", .{ green_text, output, reset_text }); print("{s}PASSED:\n{s}{s}\n\n", .{ green_text, output, reset_text });
} }
fn testing(self: *ZiglingStep, prog_node: *std.Progress.Node) ![]const u8 { fn check_test(self: *ZiglingStep, result: Child.ExecResult) !void {
print("Testing {s}...\n", .{self.exercise.main_file}); switch (result.term) {
.Exited => |code| {
const b = self.step.owner; if (code != 0) {
const exercise_path = self.exercise.main_file; // The test failed.
const path = join(b.allocator, &.{ self.work_path, exercise_path }) catch print("{s}{s}{s}\n", .{
@panic("OOM"); red_text, result.stderr, reset_text,
var zig_args = std.ArrayList([]const u8).init(b.allocator);
defer zig_args.deinit();
zig_args.append(b.zig_exe) catch @panic("OOM");
zig_args.append("test") catch @panic("OOM");
zig_args.append(b.pathFromRoot(path)) catch @panic("OOM");
const argv = zig_args.items;
var code: u8 = undefined;
_ = self.eval(argv, &code, prog_node) catch |err| {
self.printErrors();
switch (err) {
error.FileNotFound => {
print("{s}{s}: Unable to spawn the following command: file not found{s}\n", .{
red_text, self.exercise.main_file, reset_text,
}); });
dumpArgs(argv);
},
error.ExitCodeFailure => {
// Expected when test fails.
},
error.ProcessTerminated => {
print("{s}{s}: The following command terminated unexpectedly:{s}\n", .{
red_text, self.exercise.main_file, reset_text,
});
dumpArgs(argv);
},
else => {
print("{s}{s}: Unexpected error: {s}{s}\n", .{
red_text, self.exercise.main_file, @errorName(err), reset_text,
});
dumpArgs(argv);
},
}
return err; return error.TestFailed;
}; }
},
else => {
print("{s}{s} terminated unexpectedly{s}\n", .{
red_text, self.exercise.main_file, reset_text,
});
return self.result_messages; return error.UnexpectedTermination;
},
}
print("{s}PASSED{s}\n\n", .{ green_text, reset_text });
} }
fn compile(self: *ZiglingStep, prog_node: *std.Progress.Node) ![]const u8 { fn compile(self: *ZiglingStep, prog_node: *std.Progress.Node) ![]const u8 {
@ -386,7 +362,12 @@ const ZiglingStep = struct {
defer zig_args.deinit(); defer zig_args.deinit();
zig_args.append(b.zig_exe) catch @panic("OOM"); zig_args.append(b.zig_exe) catch @panic("OOM");
zig_args.append("build-exe") catch @panic("OOM");
const cmd = switch (self.exercise.kind) {
.exe => "build-exe",
.@"test" => "test",
};
zig_args.append(cmd) catch @panic("OOM");
// Enable C support for exercises that use C functions. // Enable C support for exercises that use C functions.
if (self.exercise.link_libc) { if (self.exercise.link_libc) {
@ -1220,7 +1201,7 @@ const exercises = [_]Exercise{
.{ .{
.main_file = "102_testing.zig", .main_file = "102_testing.zig",
.output = "", .output = "",
.run_test = true, .kind = .@"test",
}, },
.{ .{
.main_file = "999_the_end.zig", .main_file = "999_the_end.zig",

View file

@ -93,7 +93,7 @@ pub fn addCliTests(b: *std.Build, exercises: []const Exercise) *Step {
const case_step = createCase(b, "case-3"); const case_step = createCase(b, "case-3");
for (exercises[0 .. exercises.len - 1]) |ex| { for (exercises[0 .. exercises.len - 1]) |ex| {
if (ex.skip or ex.run_test) continue; if (ex.skip) continue;
if (ex.hint) |hint| { if (ex.hint) |hint| {
const n = ex.number(); const n = ex.number();
@ -249,21 +249,6 @@ fn check_output(step: *Step, exercise: Exercise, reader: Reader) !void {
return; return;
} }
if (exercise.run_test) {
{
const actual = try readLine(reader, &buf) orelse "EOF";
const expect = b.fmt("Testing {s}...", .{exercise.main_file});
try check(step, exercise, expect, actual);
}
{
const actual = try readLine(reader, &buf) orelse "EOF";
try check(step, exercise, "", actual);
}
return;
}
{ {
const actual = try readLine(reader, &buf) orelse "EOF"; const actual = try readLine(reader, &buf) orelse "EOF";
const expect = b.fmt("Compiling {s}...", .{exercise.main_file}); const expect = b.fmt("Compiling {s}...", .{exercise.main_file});
@ -278,12 +263,19 @@ fn check_output(step: *Step, exercise: Exercise, reader: Reader) !void {
{ {
const actual = try readLine(reader, &buf) orelse "EOF"; const actual = try readLine(reader, &buf) orelse "EOF";
const expect = "PASSED:"; const expect = switch (exercise.kind) {
.exe => "PASSED:",
.@"test" => "PASSED",
};
try check(step, exercise, expect, actual); try check(step, exercise, expect, actual);
} }
// Skip the exercise output. // Skip the exercise output.
const nlines = 1 + mem.count(u8, exercise.output, "\n") + 1; const nlines = switch (exercise.kind) {
.exe => 1 + mem.count(u8, exercise.output, "\n") + 1,
.@"test" => 1,
};
var lineno: usize = 0; var lineno: usize = 0;
while (lineno < nlines) : (lineno += 1) { while (lineno < nlines) : (lineno += 1) {
_ = try readLine(reader, &buf) orelse @panic("EOF"); _ = try readLine(reader, &buf) orelse @panic("EOF");