tests: rewrite the way to generate integration tests.

tests/cmd/main.go

@@ -8,171 +8,76 @@ import (
 	"flag"
 	"fmt"
 	"os"
-	"os/exec"
-	"strings"
 
-	"github.com/roddhjav/apparmor.d/pkg/aa"
 	"github.com/roddhjav/apparmor.d/pkg/logging"
 	"github.com/roddhjav/apparmor.d/pkg/paths"
-	"github.com/roddhjav/apparmor.d/pkg/prebuild"
-	"github.com/roddhjav/apparmor.d/tests/integration"
 )
 
-const usage = `aa-test [-h] [--bootstrap | --run | --list]
+const usage = `aa-test [-h] --bootstrap
 
     Integration tests manager tool for apparmor.d
 
 Options:
     -h, --help         Show this help message and exit.
-    -b, --bootstrap    Bootstrap tests using tldr pages.
-    -r, --run          Run a predefined list of tests.
-    -l, --list         List the configured tests.
-    -f, --file FILE    Set a tests file. Default: tests/tests.yml
-    -d, --deps         Install tests dependencies.
-    -D, --dryrun       Do not do the action, list it.
+    -b, --bootstrap    Download tests using tldr pages and generate Bats tests.
 
 `
 
 var (
 	help      bool
 	bootstrap bool
-	run       bool
-	list      bool
-	deps      bool
-	dryRun    bool
-	cfg       Config
 )
 
-type Config struct {
-	TldrDir      *paths.Path    // Default: tests/tldr
-	ScenariosDir *paths.Path    // Default: tests
-	TldrFile     *paths.Path    // Default: tests/tldr.yml
-	TestsFile    *paths.Path    // Default: tests/tests.yml
-	SettingsFile *paths.Path    // Default: tests/settings.yml
-	Profiles     paths.PathList // List of profiles
-}
-
-func NewConfig() Config {
-	cfg := Config{
-		TldrDir:      paths.New("tests/tldr"),
-		ScenariosDir: paths.New("tests/"),
-		Profiles:     paths.PathList{},
-	}
-	cfg.TldrFile = cfg.ScenariosDir.Join("tldr.yml")
-	cfg.TestsFile = cfg.ScenariosDir.Join("tests.yml")
-	cfg.SettingsFile = cfg.ScenariosDir.Join("settings.yml")
-	return cfg
-}
-
-func LoadTestSuite() (*integration.TestSuite, error) {
-	tSuite := integration.NewTestSuite()
-	if err := tSuite.ReadTests(cfg.TestsFile); err != nil {
-		return tSuite, err
-	}
-	if err := tSuite.ReadSettings(cfg.SettingsFile); err != nil {
-		return tSuite, err
-	}
-	return tSuite, nil
-}
-
 func init() {
-	cfg = NewConfig()
-	files, _ := aa.MagicRoot.ReadDir(paths.FilterOutDirectories())
-	for _, path := range files {
-		cfg.Profiles.Add(path)
-	}
-
 	flag.BoolVar(&help, "h", false, "Show this help message and exit.")
 	flag.BoolVar(&help, "help", false, "Show this help message and exit.")
-	flag.BoolVar(&bootstrap, "b", false, "Bootstrap tests using tldr pages.")
-	flag.BoolVar(&bootstrap, "bootstrap", false, "Bootstrap tests using tldr pages.")
-	flag.BoolVar(&run, "r", false, "Run a predefined list of tests.")
-	flag.BoolVar(&run, "run", false, "Run a predefined list of tests.")
-	flag.BoolVar(&list, "l", false, "List the tests to run.")
-	flag.BoolVar(&list, "list", false, "List the tests to run.")
-	flag.BoolVar(&deps, "d", false, "Install tests dependencies.")
-	flag.BoolVar(&deps, "deps", false, "Install tests dependencies.")
-	flag.BoolVar(&dryRun, "D", false, "Do not do the action, list it.")
-	flag.BoolVar(&dryRun, "dryrun", false, "Do not do the action, list it.")
+	flag.BoolVar(&bootstrap, "b", false, "Download tests using tldr pages and generate Bats tests.")
+	flag.BoolVar(&bootstrap, "bootstrap", false, "Download tests using tldr pages and generate Bats tests.")
 }
 
-func testDownload() error {
-	tldr := integration.NewTldr(cfg.TldrDir)
+type Config struct {
+	TestsDir  *paths.Path // Default: tests
+	TldrDir   *paths.Path // Default: tests/tldr
+	TldrFile  *paths.Path // Default: tests/tldr.yml
+	TestsFile *paths.Path // Default: tests/tests.yml
+	BatsDir   *paths.Path // Default: tests/bats
+}
+
+func NewConfig() *Config {
+	testsDir := paths.New("tests")
+	cfg := Config{
+		TestsDir:  testsDir,
+		TldrDir:   testsDir.Join("tldr"),
+		TldrFile:  testsDir.Join("tldr.yml"),
+		TestsFile: testsDir.Join("tldr.yml"),
+		BatsDir:   testsDir.Join("bats_dirty"),
+	}
+	return &cfg
+}
+
+func run() error {
+	logging.Step("Bootstraping tests")
+	cfg := NewConfig()
+
+	tldr := NewTldr(cfg.TldrDir)
 	if err := tldr.Download(); err != nil {
 		return err
 	}
 
-	tSuite, err := tldr.Parse()
+	tests, err := tldr.Parse()
 	if err != nil {
 		return err
 	}
+	tests = tests.Filter()
 
-	// Default bootstraped scenarios file
-	if err := tSuite.Write(cfg.TldrFile); err != nil {
-		return err
-	}
-	logging.Bullet("Default scenarios saved: %s", cfg.TldrFile)
-	logging.Bullet("Number of tests found %d", len(tSuite.Tests))
-	return nil
-}
-
-func testDeps(dryRun bool) error {
-	tSuite, err := LoadTestSuite()
-	if err != nil {
-		return nil
-	}
-
-	deps := tSuite.GetDependencies()
-	switch prebuild.Distribution {
-	case "arch":
-		arg := []string{"pacman", "-Sy", "--noconfirm"}
-		arg = append(arg, deps...)
-		cmd := exec.Command("sudo", arg...)
-		cmd.Stdout = os.Stdout
-		cmd.Stderr = os.Stderr
-		if dryRun {
-			fmt.Println(strings.Join(cmd.Args, " "))
-		} else {
-			return cmd.Run()
-		}
-	default:
-	}
-	return nil
-}
-
-func testRun(dryRun bool) error {
-	// Warning: There is no guarantee that the tests are not destructive
-	if dryRun {
-		logging.Step("List tests")
-	} else {
-		logging.Step("Run tests")
-	}
-
-	tSuite, err := LoadTestSuite()
-	if err != nil {
-		return nil
-	}
-	integration.Arguments = tSuite.Arguments
-	integration.Ignore = tSuite.Ignore
-	integration.Profiles = cfg.Profiles
-	nbCmd := 0
-	nbTest := 0
-	for _, test := range tSuite.Tests {
-		ran, nb, err := test.Run(dryRun)
-		nbTest += ran
-		nbCmd += nb
-		if err != nil {
+	for _, test := range tests {
+		if err := test.Write(cfg.BatsDir); err != nil {
 			return err
 		}
 	}
 
-	if dryRun {
-		logging.Bullet("Number of tests to run %d", nbTest)
-		logging.Bullet("Number of test commands to run %d", nbCmd)
-	} else {
-		logging.Success("Number of tests ran %d", nbTest)
-		logging.Success("Number of test command to ran %d", nbCmd)
-	}
+	logging.Bullet("Bats tests directory: %s", cfg.BatsDir)
+	logging.Bullet("Number of tests found %d", len(tests))
 	return nil
 }
 
@@ -184,18 +89,12 @@ func main() {
 		os.Exit(0)
 	}
 
-	var err error
-	if bootstrap {
-		logging.Step("Bootstraping tests")
-		err = testDownload()
-	} else if run || list {
-		err = testRun(list)
-	} else if deps {
-		err = testDeps(dryRun)
-	} else {
+	if !bootstrap {
 		flag.Usage()
 		os.Exit(1)
 	}
+
+	err := run()
 	if err != nil {
 		logging.Fatal("%s", err.Error())
 	}

tests/cmd/tests.go

@@ -0,0 +1,111 @@
+// apparmor.d - Full set of apparmor profiles
+// Copyright (C) 2023-2024 Alexandre Pujol <alexandre@pujol.io>
+// SPDX-License-Identifier: GPL-2.0-only
+
+package main
+
+import (
+	"html/template"
+	"os/exec"
+	"slices"
+	"strings"
+
+	"github.com/roddhjav/apparmor.d/pkg/aa"
+	"github.com/roddhjav/apparmor.d/pkg/paths"
+)
+
+const tmplTest = `#!/usr/bin/env bats
+# apparmor.d - Full set of apparmor profiles
+# Copyright (C) 2024 Alexandre Pujol <alexandre@pujol.io>
+# SPDX-License-Identifier: GPL-2.0-only
+{{ $name := .Name -}}
+{{ range .Commands }}
+# bats test_tags={{ $name }}
+@test "{{ $name }}: {{ .Description }}" {
+    {{ .Cmd }}
+}
+{{ end }}
+`
+
+var (
+	Profiles = getProfiles() // List of profiles in apparmor.d
+	tmpl     = template.Must(template.New("bats").Parse(tmplTest))
+)
+
+type Tests []Test
+
+// Filter returns a new list of tests with only the ones that have a profile
+func (t Tests) Filter() Tests {
+	for i := len(t) - 1; i >= 0; i-- {
+		if !t[i].HasProfile() {
+			t = slices.Delete(t, i, i+1)
+		}
+	}
+	return t
+}
+
+// Test represents of a list of tests for a given program
+type Test struct {
+	Name     string
+	Commands []Command
+}
+
+// Command is a command line to run as part of a test
+type Command struct {
+	Description string
+	Cmd         string
+}
+
+func NewTest() *Test {
+	return &Test{
+		Name:     "",
+		Commands: []Command{},
+	}
+}
+
+// HasProfile returns true if the program in the scenario is profiled in apparmor.d
+func (t *Test) HasProfile() bool {
+	return slices.Contains(Profiles, t.Name)
+}
+
+// IsInstalled returns true if the program in the scenario is installed on the system
+func (t *Test) IsInstalled() bool {
+	if _, err := exec.LookPath(t.Name); err != nil {
+		return false
+	}
+	return true
+}
+
+func (t Test) Write(dir *paths.Path) error {
+	if !t.HasProfile() {
+		return nil
+	}
+
+	path := dir.Join(t.Name + ".bats")
+	content := renderBatsFile(t)
+	if err := path.WriteFile([]byte(content)); err != nil {
+		return err
+	}
+	return nil
+}
+
+func renderBatsFile(data any) string {
+	var res strings.Builder
+	err := tmpl.Execute(&res, data)
+	if err != nil {
+		panic(err)
+	}
+	return res.String()
+}
+
+func getProfiles() []string {
+	p := []string{}
+	files, err := aa.MagicRoot.ReadDir(paths.FilterOutDirectories())
+	if err != nil {
+		panic(err)
+	}
+	for _, path := range files {
+		p = append(p, path.Base())
+	}
+	return p
+}

tests/cmd/tldr.go

@@ -0,0 +1,140 @@
+// apparmor.d - Full set of apparmor profiles
+// Copyright (C) 2023-2024 Alexandre Pujol <alexandre@pujol.io>
+// SPDX-License-Identifier: GPL-2.0-only
+
+package main
+
+import (
+	"archive/tar"
+	"compress/gzip"
+	"fmt"
+	"io"
+	"net/http"
+	"path/filepath"
+	"strings"
+
+	"github.com/roddhjav/apparmor.d/pkg/paths"
+)
+
+type Tldr struct {
+	Url    string      // Tldr download url
+	Dir    *paths.Path // Tldr cache directory
+	Ignore []string    // List of ignored software
+}
+
+func NewTldr(dir *paths.Path) Tldr {
+	return Tldr{
+		Url: "https://github.com/tldr-pages/tldr/archive/refs/heads/main.tar.gz",
+		Dir: dir,
+	}
+}
+
+// Download and extract the tldr pages into the cache directory
+func (t Tldr) Download() error {
+	gzPath := t.Dir.Parent().Join("tldr.tar.gz")
+	if !gzPath.Exist() {
+		resp, err := http.Get(t.Url)
+		if err != nil {
+			return fmt.Errorf("downloading %s: %w", t.Url, err)
+		}
+		defer resp.Body.Close()
+
+		out, err := gzPath.Create()
+		if err != nil {
+			return err
+		}
+		defer out.Close()
+
+		if _, err := io.Copy(out, resp.Body); err != nil {
+			return err
+		}
+	}
+
+	pages := []string{"tldr-main/pages/linux", "tldr-main/pages/common"}
+	return extratTo(gzPath, t.Dir, pages)
+}
+
+// Parse the tldr pages and return a list of tests
+func (t Tldr) Parse() (Tests, error) {
+	tests := make(Tests, 0)
+	files, _ := t.Dir.ReadDirRecursiveFiltered(nil, paths.FilterOutDirectories())
+	for _, path := range files {
+		content, err := path.ReadFile()
+		if err != nil {
+			return nil, err
+		}
+		raw := string(content)
+		t := Test{
+			Name:     strings.TrimSuffix(path.Base(), ".md"),
+			Commands: []Command{},
+		}
+		rawTests := strings.Split(raw, "\n-")[1:]
+		for _, test := range rawTests {
+			res := strings.Split(test, "\n")
+			dsc := strings.ReplaceAll(strings.Trim(res[0], " "), ":", "")
+			cmd := strings.Trim(strings.Trim(res[2], "`"), " ")
+			t.Commands = append(t.Commands, Command{
+				Description: dsc,
+				Cmd:         cmd,
+			})
+		}
+		tests = append(tests, t)
+	}
+	return tests, nil
+}
+
+// Either or not to extract the file
+func toExtrat(name string, subfolders []string) bool {
+	for _, subfolder := range subfolders {
+		if strings.HasPrefix(name, subfolder) {
+			return true
+		}
+	}
+	return false
+}
+
+// Extract part of an archive to a destination directory
+func extratTo(src *paths.Path, dst *paths.Path, subfolders []string) error {
+	gzIn, err := src.Open()
+	if err != nil {
+		return fmt.Errorf("opening %s: %w", src, err)
+	}
+	defer gzIn.Close()
+
+	in, err := gzip.NewReader(gzIn)
+	if err != nil {
+		return fmt.Errorf("decoding %s: %w", src, err)
+	}
+	defer in.Close()
+
+	if err := dst.MkdirAll(); err != nil {
+		return fmt.Errorf("creating %s: %w", src, err)
+	}
+
+	tarIn := tar.NewReader(in)
+	for {
+		header, err := tarIn.Next()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return err
+		}
+
+		if header.Typeflag == tar.TypeReg {
+			if !toExtrat(header.Name, subfolders) {
+				continue
+			}
+			path := dst.Join(filepath.Base(header.Name))
+			file, err := path.Create()
+			if err != nil {
+				return fmt.Errorf("creating %s: %w", file.Name(), err)
+			}
+			if _, err := io.Copy(file, tarIn); err != nil {
+				return fmt.Errorf("extracting %s: %w", file.Name(), err)
+			}
+			file.Close()
+		}
+	}
+	return nil
+}