blob: 907dc48a9a9451da689cb2892ded3374e1adbe3d [file] [log] [blame]
David Benjamin491b9212015-02-11 14:18:45 -05001/* Copyright (c) 2015, Google Inc.
2 *
3 * Permission to use, copy, modify, and/or distribute this software for any
4 * purpose with or without fee is hereby granted, provided that the above
5 * copyright notice and this permission notice appear in all copies.
6 *
7 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
8 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
9 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
10 * SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
11 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
12 * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
13 * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */
14
15package main
16
17import (
18 "bytes"
David Benjamin5f237bc2015-02-11 17:14:15 -050019 "encoding/json"
David Benjamin491b9212015-02-11 14:18:45 -050020 "flag"
21 "fmt"
22 "os"
23 "os/exec"
24 "path"
25 "strings"
David Benjamin5f237bc2015-02-11 17:14:15 -050026 "time"
David Benjamin491b9212015-02-11 14:18:45 -050027)
28
29// TODO(davidben): Link tests with the malloc shim and port -malloc-test to this runner.
30
31var (
32 useValgrind = flag.Bool("valgrind", false, "If true, run code under valgrind")
33 buildDir = flag.String("build-dir", "build", "The build directory to run the tests from.")
David Benjamin5f237bc2015-02-11 17:14:15 -050034 jsonOutput = flag.String("json-output", "", "The file to output JSON results to.")
David Benjamin491b9212015-02-11 14:18:45 -050035)
36
37type test []string
38
39var tests = []test{
40 {"crypto/base64/base64_test"},
41 {"crypto/bio/bio_test"},
42 {"crypto/bn/bn_test"},
43 {"crypto/bytestring/bytestring_test"},
44 {"crypto/cipher/aead_test", "aes-128-gcm", "crypto/cipher/test/aes_128_gcm_tests.txt"},
45 {"crypto/cipher/aead_test", "aes-128-key-wrap", "crypto/cipher/test/aes_128_key_wrap_tests.txt"},
46 {"crypto/cipher/aead_test", "aes-256-gcm", "crypto/cipher/test/aes_256_gcm_tests.txt"},
47 {"crypto/cipher/aead_test", "aes-256-key-wrap", "crypto/cipher/test/aes_256_key_wrap_tests.txt"},
48 {"crypto/cipher/aead_test", "chacha20-poly1305", "crypto/cipher/test/chacha20_poly1305_tests.txt"},
49 {"crypto/cipher/aead_test", "rc4-md5-tls", "crypto/cipher/test/rc4_md5_tls_tests.txt"},
50 {"crypto/cipher/aead_test", "rc4-sha1-tls", "crypto/cipher/test/rc4_sha1_tls_tests.txt"},
51 {"crypto/cipher/aead_test", "aes-128-cbc-sha1-tls", "crypto/cipher/test/aes_128_cbc_sha1_tls_tests.txt"},
52 {"crypto/cipher/aead_test", "aes-128-cbc-sha1-tls-implicit-iv", "crypto/cipher/test/aes_128_cbc_sha1_tls_implicit_iv_tests.txt"},
53 {"crypto/cipher/aead_test", "aes-128-cbc-sha256-tls", "crypto/cipher/test/aes_128_cbc_sha256_tls_tests.txt"},
54 {"crypto/cipher/aead_test", "aes-256-cbc-sha1-tls", "crypto/cipher/test/aes_256_cbc_sha1_tls_tests.txt"},
55 {"crypto/cipher/aead_test", "aes-256-cbc-sha1-tls-implicit-iv", "crypto/cipher/test/aes_256_cbc_sha1_tls_implicit_iv_tests.txt"},
56 {"crypto/cipher/aead_test", "aes-256-cbc-sha256-tls", "crypto/cipher/test/aes_256_cbc_sha256_tls_tests.txt"},
57 {"crypto/cipher/aead_test", "aes-256-cbc-sha384-tls", "crypto/cipher/test/aes_256_cbc_sha384_tls_tests.txt"},
58 {"crypto/cipher/aead_test", "des-ede3-cbc-sha1-tls", "crypto/cipher/test/des_ede3_cbc_sha1_tls_tests.txt"},
59 {"crypto/cipher/aead_test", "des-ede3-cbc-sha1-tls-implicit-iv", "crypto/cipher/test/des_ede3_cbc_sha1_tls_implicit_iv_tests.txt"},
60 {"crypto/cipher/aead_test", "rc4-md5-ssl3", "crypto/cipher/test/rc4_md5_ssl3_tests.txt"},
61 {"crypto/cipher/aead_test", "rc4-sha1-ssl3", "crypto/cipher/test/rc4_sha1_ssl3_tests.txt"},
62 {"crypto/cipher/aead_test", "aes-128-cbc-sha1-ssl3", "crypto/cipher/test/aes_128_cbc_sha1_ssl3_tests.txt"},
63 {"crypto/cipher/aead_test", "aes-256-cbc-sha1-ssl3", "crypto/cipher/test/aes_256_cbc_sha1_ssl3_tests.txt"},
64 {"crypto/cipher/aead_test", "des-ede3-cbc-sha1-ssl3", "crypto/cipher/test/des_ede3_cbc_sha1_ssl3_tests.txt"},
65 {"crypto/cipher/cipher_test", "crypto/cipher/test/cipher_test.txt"},
66 {"crypto/constant_time_test"},
67 {"crypto/dh/dh_test"},
68 {"crypto/digest/digest_test"},
69 {"crypto/dsa/dsa_test"},
70 {"crypto/ec/ec_test"},
71 {"crypto/ec/example_mul"},
72 {"crypto/ecdsa/ecdsa_test"},
73 {"crypto/err/err_test"},
74 {"crypto/evp/evp_test"},
75 {"crypto/evp/pbkdf_test"},
76 {"crypto/hkdf/hkdf_test"},
77 {"crypto/hmac/hmac_test"},
78 {"crypto/lhash/lhash_test"},
79 {"crypto/modes/gcm_test"},
80 {"crypto/pkcs8/pkcs12_test"},
81 {"crypto/rsa/rsa_test"},
82 {"crypto/x509/pkcs7_test"},
83 {"crypto/x509v3/tab_test"},
84 {"crypto/x509v3/v3name_test"},
85 {"ssl/pqueue/pqueue_test"},
86 {"ssl/ssl_test"},
87}
88
David Benjamin5f237bc2015-02-11 17:14:15 -050089// testOutput is a representation of Chromium's JSON test result format. See
90// https://www.chromium.org/developers/the-json-test-results-format
91type testOutput struct {
92 Version int `json:"version"`
93 Interrupted bool `json:"interrupted"`
94 PathDelimiter string `json:"path_delimiter"`
95 SecondsSinceEpoch float64 `json:"seconds_since_epoch"`
96 NumFailuresByType map[string]int `json:"num_failures_by_type"`
97 Tests map[string]testResult `json:"tests"`
98}
99
100type testResult struct {
101 Actual string `json:"actual"`
102 Expected string `json:"expected"`
103}
104
105func newTestOutput() *testOutput {
106 return &testOutput{
107 Version: 3,
108 PathDelimiter: ".",
109 SecondsSinceEpoch: float64(time.Now().UnixNano()) / float64(time.Second/time.Nanosecond),
110 NumFailuresByType: make(map[string]int),
111 Tests: make(map[string]testResult),
112 }
113}
114
115func (t *testOutput) addResult(name, result string) {
116 if _, found := t.Tests[name]; found {
117 panic(name)
118 }
119 t.Tests[name] = testResult{Actual: result, Expected: "PASS"}
120 t.NumFailuresByType[result]++
121}
122
123func (t *testOutput) writeTo(name string) error {
124 file, err := os.Create(name)
125 if err != nil {
126 return err
127 }
128 defer file.Close()
129 out, err := json.MarshalIndent(t, "", " ")
130 if err != nil {
131 return err
132 }
133 _, err = file.Write(out)
134 return err
135}
136
David Benjamin491b9212015-02-11 14:18:45 -0500137func valgrindOf(dbAttach bool, path string, args ...string) *exec.Cmd {
138 valgrindArgs := []string{"--error-exitcode=99", "--track-origins=yes", "--leak-check=full"}
139 if dbAttach {
140 valgrindArgs = append(valgrindArgs, "--db-attach=yes", "--db-command=xterm -e gdb -nw %f %p")
141 }
142 valgrindArgs = append(valgrindArgs, path)
143 valgrindArgs = append(valgrindArgs, args...)
144
145 return exec.Command("valgrind", valgrindArgs...)
146}
147
148func runTest(test test) (passed bool, err error) {
149 prog := path.Join(*buildDir, test[0])
150 args := test[1:]
151 var cmd *exec.Cmd
152 if *useValgrind {
153 cmd = valgrindOf(false, prog, args...)
154 } else {
155 cmd = exec.Command(prog, args...)
156 }
157 var stdoutBuf bytes.Buffer
158 cmd.Stdout = &stdoutBuf
159 cmd.Stderr = os.Stderr
160
161 if err := cmd.Start(); err != nil {
162 return false, err
163 }
164 if err := cmd.Wait(); err != nil {
165 return false, err
166 }
167
168 // Account for Windows line-endings.
169 stdout := bytes.Replace(stdoutBuf.Bytes(), []byte("\r\n"), []byte("\n"), -1)
170
171 if bytes.HasSuffix(stdout, []byte("PASS\n")) &&
172 (len(stdout) == 5 || stdout[len(stdout)-6] == '\n') {
173 return true, nil
174 }
175 return false, nil
176}
177
David Benjamin5f237bc2015-02-11 17:14:15 -0500178// shortTestName returns the short name of a test. It assumes that any argument
179// which ends in .txt is a path to a data file and not relevant to the test's
180// uniqueness.
181func shortTestName(test test) string {
182 var args []string
183 for _, arg := range test {
184 if !strings.HasSuffix(arg, ".txt") {
185 args = append(args, arg)
186 }
187 }
188 return strings.Join(args, " ")
189}
190
David Benjamin491b9212015-02-11 14:18:45 -0500191func main() {
192 flag.Parse()
193
David Benjamin5f237bc2015-02-11 17:14:15 -0500194 testOutput := newTestOutput()
David Benjamin491b9212015-02-11 14:18:45 -0500195 var failed []test
196 for _, test := range tests {
197 fmt.Printf("%s\n", strings.Join([]string(test), " "))
David Benjamin5f237bc2015-02-11 17:14:15 -0500198
199 name := shortTestName(test)
David Benjamin491b9212015-02-11 14:18:45 -0500200 passed, err := runTest(test)
201 if err != nil {
David Benjamin5f237bc2015-02-11 17:14:15 -0500202 fmt.Printf("%s failed to complete: %s\n", test[0], err)
David Benjamin491b9212015-02-11 14:18:45 -0500203 failed = append(failed, test)
David Benjamin5f237bc2015-02-11 17:14:15 -0500204 testOutput.addResult(name, "CRASHED")
205 } else if !passed {
David Benjamin491b9212015-02-11 14:18:45 -0500206 fmt.Printf("%s failed to print PASS on the last line.\n", test[0])
207 failed = append(failed, test)
David Benjamin5f237bc2015-02-11 17:14:15 -0500208 testOutput.addResult(name, "FAIL")
209 } else {
210 testOutput.addResult(name, "PASS")
David Benjamin491b9212015-02-11 14:18:45 -0500211 }
212 }
213
214 if len(failed) == 0 {
215 fmt.Printf("\nAll tests passed!\n")
216 } else {
217 fmt.Printf("\n%d of %d tests failed:\n", len(failed), len(tests))
218 for _, test := range failed {
219 fmt.Printf("\t%s\n", strings.Join([]string(test), " "))
220 }
221 }
David Benjamin5f237bc2015-02-11 17:14:15 -0500222
223 if *jsonOutput != "" {
224 if err := testOutput.writeTo(*jsonOutput); err != nil {
225 fmt.Fprintf(os.Stderr, "Error: %s\n", err)
226 }
227 }
David Benjamin491b9212015-02-11 14:18:45 -0500228}