David Benjamin | 491b921 | 2015-02-11 14:18:45 -0500 | [diff] [blame] | 1 | /* Copyright (c) 2015, Google Inc. |
| 2 | * |
| 3 | * Permission to use, copy, modify, and/or distribute this software for any |
| 4 | * purpose with or without fee is hereby granted, provided that the above |
| 5 | * copyright notice and this permission notice appear in all copies. |
| 6 | * |
| 7 | * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES |
| 8 | * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF |
| 9 | * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY |
| 10 | * SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES |
| 11 | * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION |
| 12 | * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN |
| 13 | * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ |
| 14 | |
| 15 | package main |
| 16 | |
| 17 | import ( |
| 18 | "bytes" |
David Benjamin | 5f237bc | 2015-02-11 17:14:15 -0500 | [diff] [blame^] | 19 | "encoding/json" |
David Benjamin | 491b921 | 2015-02-11 14:18:45 -0500 | [diff] [blame] | 20 | "flag" |
| 21 | "fmt" |
| 22 | "os" |
| 23 | "os/exec" |
| 24 | "path" |
| 25 | "strings" |
David Benjamin | 5f237bc | 2015-02-11 17:14:15 -0500 | [diff] [blame^] | 26 | "time" |
David Benjamin | 491b921 | 2015-02-11 14:18:45 -0500 | [diff] [blame] | 27 | ) |
| 28 | |
| 29 | // TODO(davidben): Link tests with the malloc shim and port -malloc-test to this runner. |
| 30 | |
| 31 | var ( |
| 32 | useValgrind = flag.Bool("valgrind", false, "If true, run code under valgrind") |
| 33 | buildDir = flag.String("build-dir", "build", "The build directory to run the tests from.") |
David Benjamin | 5f237bc | 2015-02-11 17:14:15 -0500 | [diff] [blame^] | 34 | jsonOutput = flag.String("json-output", "", "The file to output JSON results to.") |
David Benjamin | 491b921 | 2015-02-11 14:18:45 -0500 | [diff] [blame] | 35 | ) |
| 36 | |
| 37 | type test []string |
| 38 | |
| 39 | var tests = []test{ |
| 40 | {"crypto/base64/base64_test"}, |
| 41 | {"crypto/bio/bio_test"}, |
| 42 | {"crypto/bn/bn_test"}, |
| 43 | {"crypto/bytestring/bytestring_test"}, |
| 44 | {"crypto/cipher/aead_test", "aes-128-gcm", "crypto/cipher/test/aes_128_gcm_tests.txt"}, |
| 45 | {"crypto/cipher/aead_test", "aes-128-key-wrap", "crypto/cipher/test/aes_128_key_wrap_tests.txt"}, |
| 46 | {"crypto/cipher/aead_test", "aes-256-gcm", "crypto/cipher/test/aes_256_gcm_tests.txt"}, |
| 47 | {"crypto/cipher/aead_test", "aes-256-key-wrap", "crypto/cipher/test/aes_256_key_wrap_tests.txt"}, |
| 48 | {"crypto/cipher/aead_test", "chacha20-poly1305", "crypto/cipher/test/chacha20_poly1305_tests.txt"}, |
| 49 | {"crypto/cipher/aead_test", "rc4-md5-tls", "crypto/cipher/test/rc4_md5_tls_tests.txt"}, |
| 50 | {"crypto/cipher/aead_test", "rc4-sha1-tls", "crypto/cipher/test/rc4_sha1_tls_tests.txt"}, |
| 51 | {"crypto/cipher/aead_test", "aes-128-cbc-sha1-tls", "crypto/cipher/test/aes_128_cbc_sha1_tls_tests.txt"}, |
| 52 | {"crypto/cipher/aead_test", "aes-128-cbc-sha1-tls-implicit-iv", "crypto/cipher/test/aes_128_cbc_sha1_tls_implicit_iv_tests.txt"}, |
| 53 | {"crypto/cipher/aead_test", "aes-128-cbc-sha256-tls", "crypto/cipher/test/aes_128_cbc_sha256_tls_tests.txt"}, |
| 54 | {"crypto/cipher/aead_test", "aes-256-cbc-sha1-tls", "crypto/cipher/test/aes_256_cbc_sha1_tls_tests.txt"}, |
| 55 | {"crypto/cipher/aead_test", "aes-256-cbc-sha1-tls-implicit-iv", "crypto/cipher/test/aes_256_cbc_sha1_tls_implicit_iv_tests.txt"}, |
| 56 | {"crypto/cipher/aead_test", "aes-256-cbc-sha256-tls", "crypto/cipher/test/aes_256_cbc_sha256_tls_tests.txt"}, |
| 57 | {"crypto/cipher/aead_test", "aes-256-cbc-sha384-tls", "crypto/cipher/test/aes_256_cbc_sha384_tls_tests.txt"}, |
| 58 | {"crypto/cipher/aead_test", "des-ede3-cbc-sha1-tls", "crypto/cipher/test/des_ede3_cbc_sha1_tls_tests.txt"}, |
| 59 | {"crypto/cipher/aead_test", "des-ede3-cbc-sha1-tls-implicit-iv", "crypto/cipher/test/des_ede3_cbc_sha1_tls_implicit_iv_tests.txt"}, |
| 60 | {"crypto/cipher/aead_test", "rc4-md5-ssl3", "crypto/cipher/test/rc4_md5_ssl3_tests.txt"}, |
| 61 | {"crypto/cipher/aead_test", "rc4-sha1-ssl3", "crypto/cipher/test/rc4_sha1_ssl3_tests.txt"}, |
| 62 | {"crypto/cipher/aead_test", "aes-128-cbc-sha1-ssl3", "crypto/cipher/test/aes_128_cbc_sha1_ssl3_tests.txt"}, |
| 63 | {"crypto/cipher/aead_test", "aes-256-cbc-sha1-ssl3", "crypto/cipher/test/aes_256_cbc_sha1_ssl3_tests.txt"}, |
| 64 | {"crypto/cipher/aead_test", "des-ede3-cbc-sha1-ssl3", "crypto/cipher/test/des_ede3_cbc_sha1_ssl3_tests.txt"}, |
| 65 | {"crypto/cipher/cipher_test", "crypto/cipher/test/cipher_test.txt"}, |
| 66 | {"crypto/constant_time_test"}, |
| 67 | {"crypto/dh/dh_test"}, |
| 68 | {"crypto/digest/digest_test"}, |
| 69 | {"crypto/dsa/dsa_test"}, |
| 70 | {"crypto/ec/ec_test"}, |
| 71 | {"crypto/ec/example_mul"}, |
| 72 | {"crypto/ecdsa/ecdsa_test"}, |
| 73 | {"crypto/err/err_test"}, |
| 74 | {"crypto/evp/evp_test"}, |
| 75 | {"crypto/evp/pbkdf_test"}, |
| 76 | {"crypto/hkdf/hkdf_test"}, |
| 77 | {"crypto/hmac/hmac_test"}, |
| 78 | {"crypto/lhash/lhash_test"}, |
| 79 | {"crypto/modes/gcm_test"}, |
| 80 | {"crypto/pkcs8/pkcs12_test"}, |
| 81 | {"crypto/rsa/rsa_test"}, |
| 82 | {"crypto/x509/pkcs7_test"}, |
| 83 | {"crypto/x509v3/tab_test"}, |
| 84 | {"crypto/x509v3/v3name_test"}, |
| 85 | {"ssl/pqueue/pqueue_test"}, |
| 86 | {"ssl/ssl_test"}, |
| 87 | } |
| 88 | |
David Benjamin | 5f237bc | 2015-02-11 17:14:15 -0500 | [diff] [blame^] | 89 | // testOutput is a representation of Chromium's JSON test result format. See |
| 90 | // https://www.chromium.org/developers/the-json-test-results-format |
| 91 | type testOutput struct { |
| 92 | Version int `json:"version"` |
| 93 | Interrupted bool `json:"interrupted"` |
| 94 | PathDelimiter string `json:"path_delimiter"` |
| 95 | SecondsSinceEpoch float64 `json:"seconds_since_epoch"` |
| 96 | NumFailuresByType map[string]int `json:"num_failures_by_type"` |
| 97 | Tests map[string]testResult `json:"tests"` |
| 98 | } |
| 99 | |
| 100 | type testResult struct { |
| 101 | Actual string `json:"actual"` |
| 102 | Expected string `json:"expected"` |
| 103 | } |
| 104 | |
| 105 | func newTestOutput() *testOutput { |
| 106 | return &testOutput{ |
| 107 | Version: 3, |
| 108 | PathDelimiter: ".", |
| 109 | SecondsSinceEpoch: float64(time.Now().UnixNano()) / float64(time.Second/time.Nanosecond), |
| 110 | NumFailuresByType: make(map[string]int), |
| 111 | Tests: make(map[string]testResult), |
| 112 | } |
| 113 | } |
| 114 | |
| 115 | func (t *testOutput) addResult(name, result string) { |
| 116 | if _, found := t.Tests[name]; found { |
| 117 | panic(name) |
| 118 | } |
| 119 | t.Tests[name] = testResult{Actual: result, Expected: "PASS"} |
| 120 | t.NumFailuresByType[result]++ |
| 121 | } |
| 122 | |
| 123 | func (t *testOutput) writeTo(name string) error { |
| 124 | file, err := os.Create(name) |
| 125 | if err != nil { |
| 126 | return err |
| 127 | } |
| 128 | defer file.Close() |
| 129 | out, err := json.MarshalIndent(t, "", " ") |
| 130 | if err != nil { |
| 131 | return err |
| 132 | } |
| 133 | _, err = file.Write(out) |
| 134 | return err |
| 135 | } |
| 136 | |
David Benjamin | 491b921 | 2015-02-11 14:18:45 -0500 | [diff] [blame] | 137 | func valgrindOf(dbAttach bool, path string, args ...string) *exec.Cmd { |
| 138 | valgrindArgs := []string{"--error-exitcode=99", "--track-origins=yes", "--leak-check=full"} |
| 139 | if dbAttach { |
| 140 | valgrindArgs = append(valgrindArgs, "--db-attach=yes", "--db-command=xterm -e gdb -nw %f %p") |
| 141 | } |
| 142 | valgrindArgs = append(valgrindArgs, path) |
| 143 | valgrindArgs = append(valgrindArgs, args...) |
| 144 | |
| 145 | return exec.Command("valgrind", valgrindArgs...) |
| 146 | } |
| 147 | |
| 148 | func runTest(test test) (passed bool, err error) { |
| 149 | prog := path.Join(*buildDir, test[0]) |
| 150 | args := test[1:] |
| 151 | var cmd *exec.Cmd |
| 152 | if *useValgrind { |
| 153 | cmd = valgrindOf(false, prog, args...) |
| 154 | } else { |
| 155 | cmd = exec.Command(prog, args...) |
| 156 | } |
| 157 | var stdoutBuf bytes.Buffer |
| 158 | cmd.Stdout = &stdoutBuf |
| 159 | cmd.Stderr = os.Stderr |
| 160 | |
| 161 | if err := cmd.Start(); err != nil { |
| 162 | return false, err |
| 163 | } |
| 164 | if err := cmd.Wait(); err != nil { |
| 165 | return false, err |
| 166 | } |
| 167 | |
| 168 | // Account for Windows line-endings. |
| 169 | stdout := bytes.Replace(stdoutBuf.Bytes(), []byte("\r\n"), []byte("\n"), -1) |
| 170 | |
| 171 | if bytes.HasSuffix(stdout, []byte("PASS\n")) && |
| 172 | (len(stdout) == 5 || stdout[len(stdout)-6] == '\n') { |
| 173 | return true, nil |
| 174 | } |
| 175 | return false, nil |
| 176 | } |
| 177 | |
David Benjamin | 5f237bc | 2015-02-11 17:14:15 -0500 | [diff] [blame^] | 178 | // shortTestName returns the short name of a test. It assumes that any argument |
| 179 | // which ends in .txt is a path to a data file and not relevant to the test's |
| 180 | // uniqueness. |
| 181 | func shortTestName(test test) string { |
| 182 | var args []string |
| 183 | for _, arg := range test { |
| 184 | if !strings.HasSuffix(arg, ".txt") { |
| 185 | args = append(args, arg) |
| 186 | } |
| 187 | } |
| 188 | return strings.Join(args, " ") |
| 189 | } |
| 190 | |
David Benjamin | 491b921 | 2015-02-11 14:18:45 -0500 | [diff] [blame] | 191 | func main() { |
| 192 | flag.Parse() |
| 193 | |
David Benjamin | 5f237bc | 2015-02-11 17:14:15 -0500 | [diff] [blame^] | 194 | testOutput := newTestOutput() |
David Benjamin | 491b921 | 2015-02-11 14:18:45 -0500 | [diff] [blame] | 195 | var failed []test |
| 196 | for _, test := range tests { |
| 197 | fmt.Printf("%s\n", strings.Join([]string(test), " ")) |
David Benjamin | 5f237bc | 2015-02-11 17:14:15 -0500 | [diff] [blame^] | 198 | |
| 199 | name := shortTestName(test) |
David Benjamin | 491b921 | 2015-02-11 14:18:45 -0500 | [diff] [blame] | 200 | passed, err := runTest(test) |
| 201 | if err != nil { |
David Benjamin | 5f237bc | 2015-02-11 17:14:15 -0500 | [diff] [blame^] | 202 | fmt.Printf("%s failed to complete: %s\n", test[0], err) |
David Benjamin | 491b921 | 2015-02-11 14:18:45 -0500 | [diff] [blame] | 203 | failed = append(failed, test) |
David Benjamin | 5f237bc | 2015-02-11 17:14:15 -0500 | [diff] [blame^] | 204 | testOutput.addResult(name, "CRASHED") |
| 205 | } else if !passed { |
David Benjamin | 491b921 | 2015-02-11 14:18:45 -0500 | [diff] [blame] | 206 | fmt.Printf("%s failed to print PASS on the last line.\n", test[0]) |
| 207 | failed = append(failed, test) |
David Benjamin | 5f237bc | 2015-02-11 17:14:15 -0500 | [diff] [blame^] | 208 | testOutput.addResult(name, "FAIL") |
| 209 | } else { |
| 210 | testOutput.addResult(name, "PASS") |
David Benjamin | 491b921 | 2015-02-11 14:18:45 -0500 | [diff] [blame] | 211 | } |
| 212 | } |
| 213 | |
| 214 | if len(failed) == 0 { |
| 215 | fmt.Printf("\nAll tests passed!\n") |
| 216 | } else { |
| 217 | fmt.Printf("\n%d of %d tests failed:\n", len(failed), len(tests)) |
| 218 | for _, test := range failed { |
| 219 | fmt.Printf("\t%s\n", strings.Join([]string(test), " ")) |
| 220 | } |
| 221 | } |
David Benjamin | 5f237bc | 2015-02-11 17:14:15 -0500 | [diff] [blame^] | 222 | |
| 223 | if *jsonOutput != "" { |
| 224 | if err := testOutput.writeTo(*jsonOutput); err != nil { |
| 225 | fmt.Fprintf(os.Stderr, "Error: %s\n", err) |
| 226 | } |
| 227 | } |
David Benjamin | 491b921 | 2015-02-11 14:18:45 -0500 | [diff] [blame] | 228 | } |