diff --git a/go.mod b/go.mod index bec7213d7..085850645 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,6 @@ go 1.21 require ( github.com/stretchr/testify v1.9.0 - gopkg.in/yaml.v3 v3.0.1 ) require ( diff --git a/tests/cmd/main.go b/tests/cmd/main.go index b549aab34..19dd1cf0d 100644 --- a/tests/cmd/main.go +++ b/tests/cmd/main.go @@ -8,171 +8,76 @@ import ( "flag" "fmt" "os" - "os/exec" - "strings" - "github.com/roddhjav/apparmor.d/pkg/aa" "github.com/roddhjav/apparmor.d/pkg/logging" "github.com/roddhjav/apparmor.d/pkg/paths" - "github.com/roddhjav/apparmor.d/pkg/prebuild" - "github.com/roddhjav/apparmor.d/tests/integration" ) -const usage = `aa-test [-h] [--bootstrap | --run | --list] +const usage = `aa-test [-h] --bootstrap Integration tests manager tool for apparmor.d Options: -h, --help Show this help message and exit. - -b, --bootstrap Bootstrap tests using tldr pages. - -r, --run Run a predefined list of tests. - -l, --list List the configured tests. - -f, --file FILE Set a tests file. Default: tests/tests.yml - -d, --deps Install tests dependencies. - -D, --dryrun Do not do the action, list it. + -b, --bootstrap Download tests using tldr pages and generate Bats tests. ` var ( help bool bootstrap bool - run bool - list bool - deps bool - dryRun bool - cfg Config ) -type Config struct { - TldrDir *paths.Path // Default: tests/tldr - ScenariosDir *paths.Path // Default: tests - TldrFile *paths.Path // Default: tests/tldr.yml - TestsFile *paths.Path // Default: tests/tests.yml - SettingsFile *paths.Path // Default: tests/settings.yml - Profiles paths.PathList // List of profiles -} - -func NewConfig() Config { - cfg := Config{ - TldrDir: paths.New("tests/tldr"), - ScenariosDir: paths.New("tests/"), - Profiles: paths.PathList{}, - } - cfg.TldrFile = cfg.ScenariosDir.Join("tldr.yml") - cfg.TestsFile = cfg.ScenariosDir.Join("tests.yml") - cfg.SettingsFile = cfg.ScenariosDir.Join("settings.yml") - return cfg -} - -func LoadTestSuite() (*integration.TestSuite, error) { - tSuite := integration.NewTestSuite() - if err := tSuite.ReadTests(cfg.TestsFile); err != nil { - return tSuite, err - } - if err := tSuite.ReadSettings(cfg.SettingsFile); err != nil { - return tSuite, err - } - return tSuite, nil -} - func init() { - cfg = NewConfig() - files, _ := aa.MagicRoot.ReadDir(paths.FilterOutDirectories()) - for _, path := range files { - cfg.Profiles.Add(path) - } - flag.BoolVar(&help, "h", false, "Show this help message and exit.") flag.BoolVar(&help, "help", false, "Show this help message and exit.") - flag.BoolVar(&bootstrap, "b", false, "Bootstrap tests using tldr pages.") - flag.BoolVar(&bootstrap, "bootstrap", false, "Bootstrap tests using tldr pages.") - flag.BoolVar(&run, "r", false, "Run a predefined list of tests.") - flag.BoolVar(&run, "run", false, "Run a predefined list of tests.") - flag.BoolVar(&list, "l", false, "List the tests to run.") - flag.BoolVar(&list, "list", false, "List the tests to run.") - flag.BoolVar(&deps, "d", false, "Install tests dependencies.") - flag.BoolVar(&deps, "deps", false, "Install tests dependencies.") - flag.BoolVar(&dryRun, "D", false, "Do not do the action, list it.") - flag.BoolVar(&dryRun, "dryrun", false, "Do not do the action, list it.") + flag.BoolVar(&bootstrap, "b", false, "Download tests using tldr pages and generate Bats tests.") + flag.BoolVar(&bootstrap, "bootstrap", false, "Download tests using tldr pages and generate Bats tests.") } -func testDownload() error { - tldr := integration.NewTldr(cfg.TldrDir) +type Config struct { + TestsDir *paths.Path // Default: tests + TldrDir *paths.Path // Default: tests/tldr + TldrFile *paths.Path // Default: tests/tldr.yml + TestsFile *paths.Path // Default: tests/tests.yml + BatsDir *paths.Path // Default: tests/bats +} + +func NewConfig() *Config { + testsDir := paths.New("tests") + cfg := Config{ + TestsDir: testsDir, + TldrDir: testsDir.Join("tldr"), + TldrFile: testsDir.Join("tldr.yml"), + TestsFile: testsDir.Join("tldr.yml"), + BatsDir: testsDir.Join("bats_dirty"), + } + return &cfg +} + +func run() error { + logging.Step("Bootstraping tests") + cfg := NewConfig() + + tldr := NewTldr(cfg.TldrDir) if err := tldr.Download(); err != nil { return err } - tSuite, err := tldr.Parse() + tests, err := tldr.Parse() if err != nil { return err } + tests = tests.Filter() - // Default bootstraped scenarios file - if err := tSuite.Write(cfg.TldrFile); err != nil { - return err - } - logging.Bullet("Default scenarios saved: %s", cfg.TldrFile) - logging.Bullet("Number of tests found %d", len(tSuite.Tests)) - return nil -} - -func testDeps(dryRun bool) error { - tSuite, err := LoadTestSuite() - if err != nil { - return nil - } - - deps := tSuite.GetDependencies() - switch prebuild.Distribution { - case "arch": - arg := []string{"pacman", "-Sy", "--noconfirm"} - arg = append(arg, deps...) - cmd := exec.Command("sudo", arg...) - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - if dryRun { - fmt.Println(strings.Join(cmd.Args, " ")) - } else { - return cmd.Run() - } - default: - } - return nil -} - -func testRun(dryRun bool) error { - // Warning: There is no guarantee that the tests are not destructive - if dryRun { - logging.Step("List tests") - } else { - logging.Step("Run tests") - } - - tSuite, err := LoadTestSuite() - if err != nil { - return nil - } - integration.Arguments = tSuite.Arguments - integration.Ignore = tSuite.Ignore - integration.Profiles = cfg.Profiles - nbCmd := 0 - nbTest := 0 - for _, test := range tSuite.Tests { - ran, nb, err := test.Run(dryRun) - nbTest += ran - nbCmd += nb - if err != nil { + for _, test := range tests { + if err := test.Write(cfg.BatsDir); err != nil { return err } } - if dryRun { - logging.Bullet("Number of tests to run %d", nbTest) - logging.Bullet("Number of test commands to run %d", nbCmd) - } else { - logging.Success("Number of tests ran %d", nbTest) - logging.Success("Number of test command to ran %d", nbCmd) - } + logging.Bullet("Bats tests directory: %s", cfg.BatsDir) + logging.Bullet("Number of tests found %d", len(tests)) return nil } @@ -184,18 +89,12 @@ func main() { os.Exit(0) } - var err error - if bootstrap { - logging.Step("Bootstraping tests") - err = testDownload() - } else if run || list { - err = testRun(list) - } else if deps { - err = testDeps(dryRun) - } else { + if !bootstrap { flag.Usage() os.Exit(1) } + + err := run() if err != nil { logging.Fatal("%s", err.Error()) } diff --git a/tests/cmd/tests.go b/tests/cmd/tests.go new file mode 100644 index 000000000..d145fbb07 --- /dev/null +++ b/tests/cmd/tests.go @@ -0,0 +1,111 @@ +// apparmor.d - Full set of apparmor profiles +// Copyright (C) 2023-2024 Alexandre Pujol +// SPDX-License-Identifier: GPL-2.0-only + +package main + +import ( + "html/template" + "os/exec" + "slices" + "strings" + + "github.com/roddhjav/apparmor.d/pkg/aa" + "github.com/roddhjav/apparmor.d/pkg/paths" +) + +const tmplTest = `#!/usr/bin/env bats +# apparmor.d - Full set of apparmor profiles +# Copyright (C) 2024 Alexandre Pujol +# SPDX-License-Identifier: GPL-2.0-only +{{ $name := .Name -}} +{{ range .Commands }} +# bats test_tags={{ $name }} +@test "{{ $name }}: {{ .Description }}" { + {{ .Cmd }} +} +{{ end }} +` + +var ( + Profiles = getProfiles() // List of profiles in apparmor.d + tmpl = template.Must(template.New("bats").Parse(tmplTest)) +) + +type Tests []Test + +// Filter returns a new list of tests with only the ones that have a profile +func (t Tests) Filter() Tests { + for i := len(t) - 1; i >= 0; i-- { + if !t[i].HasProfile() { + t = slices.Delete(t, i, i+1) + } + } + return t +} + +// Test represents of a list of tests for a given program +type Test struct { + Name string + Commands []Command +} + +// Command is a command line to run as part of a test +type Command struct { + Description string + Cmd string +} + +func NewTest() *Test { + return &Test{ + Name: "", + Commands: []Command{}, + } +} + +// HasProfile returns true if the program in the scenario is profiled in apparmor.d +func (t *Test) HasProfile() bool { + return slices.Contains(Profiles, t.Name) +} + +// IsInstalled returns true if the program in the scenario is installed on the system +func (t *Test) IsInstalled() bool { + if _, err := exec.LookPath(t.Name); err != nil { + return false + } + return true +} + +func (t Test) Write(dir *paths.Path) error { + if !t.HasProfile() { + return nil + } + + path := dir.Join(t.Name + ".bats") + content := renderBatsFile(t) + if err := path.WriteFile([]byte(content)); err != nil { + return err + } + return nil +} + +func renderBatsFile(data any) string { + var res strings.Builder + err := tmpl.Execute(&res, data) + if err != nil { + panic(err) + } + return res.String() +} + +func getProfiles() []string { + p := []string{} + files, err := aa.MagicRoot.ReadDir(paths.FilterOutDirectories()) + if err != nil { + panic(err) + } + for _, path := range files { + p = append(p, path.Base()) + } + return p +} diff --git a/tests/integration/tldr.go b/tests/cmd/tldr.go similarity index 52% rename from tests/integration/tldr.go rename to tests/cmd/tldr.go index fb879d15e..d86c80565 100644 --- a/tests/integration/tldr.go +++ b/tests/cmd/tldr.go @@ -2,12 +2,15 @@ // Copyright (C) 2023-2024 Alexandre Pujol // SPDX-License-Identifier: GPL-2.0-only -package integration +package main import ( + "archive/tar" + "compress/gzip" "fmt" "io" "net/http" + "path/filepath" "strings" "github.com/roddhjav/apparmor.d/pkg/paths" @@ -51,9 +54,9 @@ func (t Tldr) Download() error { return extratTo(gzPath, t.Dir, pages) } -// Parse the tldr pages and return a list of scenarios -func (t Tldr) Parse() (*TestSuite, error) { - testSuite := NewTestSuite() +// Parse the tldr pages and return a list of tests +func (t Tldr) Parse() (Tests, error) { + tests := make(Tests, 0) files, _ := t.Dir.ReadDirRecursiveFiltered(nil, paths.FilterOutDirectories()) for _, path := range files { content, err := path.ReadFile() @@ -61,29 +64,77 @@ func (t Tldr) Parse() (*TestSuite, error) { return nil, err } raw := string(content) - t := &Test{ - Name: strings.TrimSuffix(path.Base(), ".md"), - Root: false, - Arguments: map[string]string{}, - Commands: []Command{}, - } - if strings.Contains(raw, "sudo") { - t.Root = true + t := Test{ + Name: strings.TrimSuffix(path.Base(), ".md"), + Commands: []Command{}, } rawTests := strings.Split(raw, "\n-")[1:] for _, test := range rawTests { res := strings.Split(test, "\n") dsc := strings.ReplaceAll(strings.Trim(res[0], " "), ":", "") cmd := strings.Trim(strings.Trim(res[2], "`"), " ") - if t.Root { - cmd = strings.ReplaceAll(cmd, "sudo ", "") - } t.Commands = append(t.Commands, Command{ Description: dsc, Cmd: cmd, }) } - testSuite.Tests = append(testSuite.Tests, *t) + tests = append(tests, t) } - return testSuite, nil + return tests, nil +} + +// Either or not to extract the file +func toExtrat(name string, subfolders []string) bool { + for _, subfolder := range subfolders { + if strings.HasPrefix(name, subfolder) { + return true + } + } + return false +} + +// Extract part of an archive to a destination directory +func extratTo(src *paths.Path, dst *paths.Path, subfolders []string) error { + gzIn, err := src.Open() + if err != nil { + return fmt.Errorf("opening %s: %w", src, err) + } + defer gzIn.Close() + + in, err := gzip.NewReader(gzIn) + if err != nil { + return fmt.Errorf("decoding %s: %w", src, err) + } + defer in.Close() + + if err := dst.MkdirAll(); err != nil { + return fmt.Errorf("creating %s: %w", src, err) + } + + tarIn := tar.NewReader(in) + for { + header, err := tarIn.Next() + if err == io.EOF { + break + } + if err != nil { + return err + } + + if header.Typeflag == tar.TypeReg { + if !toExtrat(header.Name, subfolders) { + continue + } + path := dst.Join(filepath.Base(header.Name)) + file, err := path.Create() + if err != nil { + return fmt.Errorf("creating %s: %w", file.Name(), err) + } + if _, err := io.Copy(file, tarIn); err != nil { + return fmt.Errorf("extracting %s: %w", file.Name(), err) + } + file.Close() + } + } + return nil } diff --git a/tests/integration/paths.go b/tests/integration/paths.go deleted file mode 100644 index 8d4a1cc9c..000000000 --- a/tests/integration/paths.go +++ /dev/null @@ -1,72 +0,0 @@ -// apparmor.d - Full set of apparmor profiles -// Copyright (C) 2023-2024 Alexandre Pujol -// SPDX-License-Identifier: GPL-2.0-only - -package integration - -import ( - "archive/tar" - "compress/gzip" - "fmt" - "io" - "path/filepath" - "strings" - - "github.com/roddhjav/apparmor.d/pkg/paths" -) - -// Either or not to extract the file -func toExtrat(name string, subfolders []string) bool { - for _, subfolder := range subfolders { - if strings.HasPrefix(name, subfolder) { - return true - } - } - return false -} - -// Extract part of an archive to a destination directory -func extratTo(src *paths.Path, dst *paths.Path, subfolders []string) error { - gzIn, err := src.Open() - if err != nil { - return fmt.Errorf("opening %s: %w", src, err) - } - defer gzIn.Close() - - in, err := gzip.NewReader(gzIn) - if err != nil { - return fmt.Errorf("decoding %s: %w", src, err) - } - defer in.Close() - - if err := dst.MkdirAll(); err != nil { - return fmt.Errorf("creating %s: %w", src, err) - } - - tarIn := tar.NewReader(in) - for { - header, err := tarIn.Next() - if err == io.EOF { - break - } - if err != nil { - return err - } - - if header.Typeflag == tar.TypeReg { - if !toExtrat(header.Name, subfolders) { - continue - } - path := dst.Join(filepath.Base(header.Name)) - file, err := path.Create() - if err != nil { - return fmt.Errorf("creating %s: %w", file.Name(), err) - } - if _, err := io.Copy(file, tarIn); err != nil { - return fmt.Errorf("extracting %s: %w", file.Name(), err) - } - file.Close() - } - } - return nil -} diff --git a/tests/integration/scenario.go b/tests/integration/scenario.go deleted file mode 100644 index 94e9a728f..000000000 --- a/tests/integration/scenario.go +++ /dev/null @@ -1,141 +0,0 @@ -// apparmor.d - Full set of apparmor profiles -// Copyright (C) 2023-2024 Alexandre Pujol -// SPDX-License-Identifier: GPL-2.0-only - -// TODO: -// - Finish templating -// - Provide a large selection of resources: files, disks, http server... for automatic test on them -// - Expand support for interactive program (stdin and Control-D) -// - Properlly log the test result -// - Dbus integration - -package integration - -import ( - "bytes" - "fmt" - "io" - "os/exec" - "strings" - - "github.com/roddhjav/apparmor.d/pkg/logging" - "github.com/roddhjav/apparmor.d/pkg/paths" -) - -var ( - Ignore []string // Do not run some scenarios - Arguments map[string]string // Common arguments used across all scenarios - Profiles paths.PathList // List of profiles in apparmor.d -) - -// Test represents of a list of tests for a given program -type Test struct { - Name string `yaml:"name"` - Root bool `yaml:"root"` // Run the test as user or as root - Dependencies []string `yaml:"require"` // Packages required for the tests to run "$(pacman -Qqo Scenario.Name)" - Arguments map[string]string `yaml:"arguments"` // Arguments to pass to the program, specific to this scenario - Commands []Command `yaml:"tests"` -} - -// Command is a command line to run as part of a test -type Command struct { - Description string `yaml:"dsc"` - Cmd string `yaml:"cmd"` - Stdin []string `yaml:"stdin"` -} - -func NewTest() *Test { - return &Test{ - Name: "", - Root: false, - Dependencies: []string{}, - Arguments: map[string]string{}, - Commands: []Command{}, - } -} - -// HasProfile returns true if the program in the scenario is profiled in apparmor.d -func (t *Test) HasProfile() bool { - for _, path := range Profiles { - if t.Name == path.Base() { - return true - } - } - return false -} - -// IsInstalled returns true if the program in the scenario is installed on the system -func (t *Test) IsInstalled() bool { - if _, err := exec.LookPath(t.Name); err != nil { - return false - } - return true -} - -func (t *Test) resolve(in string) string { - res := in - for key, value := range t.Arguments { - res = strings.ReplaceAll(res, "{{ "+key+" }}", value) - } - return res -} - -// mergeArguments merge the arguments of the scenario with the global arguments -// Test arguments have priority over global arguments -func (t *Test) mergeArguments(args map[string]string) { - if len(t.Arguments) == 0 { - t.Arguments = map[string]string{} - } - for key, value := range args { - t.Arguments[key] = value - } -} - -// Run the scenarios tests -func (t *Test) Run(dryRun bool) (ran int, nb int, err error) { - nb = 0 - if t.HasProfile() && t.IsInstalled() { - logging.Step("%s", t.Name) - t.mergeArguments(Arguments) - for _, test := range t.Commands { - cmd := t.resolve(test.Cmd) - if !strings.Contains(cmd, "{{") { - nb++ - if dryRun { - logging.Bullet("%s", cmd) - } else { - cmdErr := t.run(cmd, strings.Join(test.Stdin, "\n")) - if cmdErr != nil { - logging.Error("%v", cmdErr) - } else { - logging.Success("%s", cmd) - } - } - } - } - return 1, nb, err - } - return 0, nb, err -} - -func (t *Test) run(cmdline string, in string) error { - var testErr bytes.Buffer - - // Running the command in a shell ensure it does not run confined under the sudo profile. - // The shell is run unconfined and therefore the cmdline can be confined without no-new-privs issue. - sufix := " &" // TODO: we need a goroutine here - cmd := exec.Command("sh", "-c", cmdline+sufix) - if t.Root { - cmd = exec.Command("sudo", "sh", "-c", cmdline+sufix) - } - - stderr := io.MultiWriter(Stderr, &testErr) - cmd.Stdin = strings.NewReader(in) - cmd.Stdout = Stdout - cmd.Stderr = stderr - err := cmd.Run() - if testErr.Len() > 0 { - return fmt.Errorf("%s", testErr.String()) - } - return err -} diff --git a/tests/integration/suite.go b/tests/integration/suite.go deleted file mode 100644 index 26ef24994..000000000 --- a/tests/integration/suite.go +++ /dev/null @@ -1,114 +0,0 @@ -// apparmor.d - Full set of apparmor profiles -// Copyright (C) 2023-2024 Alexandre Pujol -// SPDX-License-Identifier: GPL-2.0-only - -package integration - -import ( - "os" - - "github.com/roddhjav/apparmor.d/pkg/logs" - "github.com/roddhjav/apparmor.d/pkg/paths" - "github.com/roddhjav/apparmor.d/pkg/util" - "gopkg.in/yaml.v3" -) - -var ( - // Integration tests standard output - Stdout *os.File - - // Integration tests standard error output - Stderr *os.File - - stdoutPath = paths.New("tests/out.log") - stderrPath = paths.New("tests/err.log") -) - -// TestSuite is the apparmod.d integration tests to run -type TestSuite struct { - Tests []Test // List of tests to run - Ignore []string // Do not run some tests - Arguments map[string]string // Common arguments used across all tests -} - -// NewScenarios returns a new list of scenarios -func NewTestSuite() *TestSuite { - var err error - Stdout, err = stdoutPath.Create() - if err != nil { - panic(err) - } - Stderr, err = stderrPath.Create() - if err != nil { - panic(err) - } - return &TestSuite{ - Tests: []Test{}, - Ignore: []string{}, - Arguments: map[string]string{}, - } -} - -// Write export the list of scenarios to a file -func (t *TestSuite) Write(path *paths.Path) error { - jsonString, err := yaml.Marshal(&t.Tests) - if err != nil { - return err - } - - path = path.Clean() - file, err := path.Create() - if err != nil { - return err - } - defer file.Close() - - // Cleanup a bit - res := string(jsonString) - regClean := util.ToRegexRepl([]string{ - "- name:", "\n- name:", - `(?m)^.*stdin: \[\].*$`, ``, - `{{`, `{{ `, - `}}`, ` }}`, - }) - res = regClean.Replace(res) - _, err = file.WriteString("---\n" + res) - return err -} - -// ReadTests import the tests from a file -func (t *TestSuite) ReadTests(path *paths.Path) error { - content, _ := path.ReadFile() - return yaml.Unmarshal(content, &t.Tests) -} - -// ReadSettings import the common argument and ignore list from a file -func (t *TestSuite) ReadSettings(path *paths.Path) error { - type temp struct { - Arguments map[string]string `yaml:"arguments"` - Ignore []string `yaml:"ignore"` - } - tmp := temp{} - content, _ := path.ReadFile() - if err := yaml.Unmarshal(content, &tmp); err != nil { - return err - } - t.Arguments = tmp.Arguments - t.Ignore = tmp.Ignore - return nil -} - -// Results returns a sum up of the apparmor logs raised by the scenarios -func (t *TestSuite) Results() string { - file, _ := logs.GetAuditLogs(logs.LogFiles[0]) - aaLogs := logs.New(file, "") - return aaLogs.String() -} - -func (t *TestSuite) GetDependencies() []string { - res := []string{} - for _, test := range t.Tests { - res = append(res, test.Dependencies...) - } - return res -}