| Dan Williams | f380420 | 2018-01-29 17:02:22 -0800 | [diff] [blame^] | 1 | // SPDX-License-Identifier: GPL-2.0 |
| 2 | // Copyright(c) 2018 Linus Torvalds. All rights reserved. |
| 3 | // Copyright(c) 2018 Alexei Starovoitov. All rights reserved. |
| 4 | // Copyright(c) 2018 Intel Corporation. All rights reserved. |
| 5 | |
| 6 | #ifndef _LINUX_NOSPEC_H |
| 7 | #define _LINUX_NOSPEC_H |
| 8 | |
| 9 | /** |
| 10 | * array_index_mask_nospec() - generate a ~0 mask when index < size, 0 otherwise |
| 11 | * @index: array element index |
| 12 | * @size: number of elements in array |
| 13 | * |
| 14 | * When @index is out of bounds (@index >= @size), the sign bit will be |
| 15 | * set. Extend the sign bit to all bits and invert, giving a result of |
| 16 | * zero for an out of bounds index, or ~0 if within bounds [0, @size). |
| 17 | */ |
| 18 | #ifndef array_index_mask_nospec |
| 19 | static inline unsigned long array_index_mask_nospec(unsigned long index, |
| 20 | unsigned long size) |
| 21 | { |
| 22 | /* |
| 23 | * Warn developers about inappropriate array_index_nospec() usage. |
| 24 | * |
| 25 | * Even if the CPU speculates past the WARN_ONCE branch, the |
| 26 | * sign bit of @index is taken into account when generating the |
| 27 | * mask. |
| 28 | * |
| 29 | * This warning is compiled out when the compiler can infer that |
| 30 | * @index and @size are less than LONG_MAX. |
| 31 | */ |
| 32 | if (WARN_ONCE(index > LONG_MAX || size > LONG_MAX, |
| 33 | "array_index_nospec() limited to range of [0, LONG_MAX]\n")) |
| 34 | return 0; |
| 35 | |
| 36 | /* |
| 37 | * Always calculate and emit the mask even if the compiler |
| 38 | * thinks the mask is not needed. The compiler does not take |
| 39 | * into account the value of @index under speculation. |
| 40 | */ |
| 41 | OPTIMIZER_HIDE_VAR(index); |
| 42 | return ~(long)(index | (size - 1UL - index)) >> (BITS_PER_LONG - 1); |
| 43 | } |
| 44 | #endif |
| 45 | |
| 46 | /* |
| 47 | * array_index_nospec - sanitize an array index after a bounds check |
| 48 | * |
| 49 | * For a code sequence like: |
| 50 | * |
| 51 | * if (index < size) { |
| 52 | * index = array_index_nospec(index, size); |
| 53 | * val = array[index]; |
| 54 | * } |
| 55 | * |
| 56 | * ...if the CPU speculates past the bounds check then |
| 57 | * array_index_nospec() will clamp the index within the range of [0, |
| 58 | * size). |
| 59 | */ |
| 60 | #define array_index_nospec(index, size) \ |
| 61 | ({ \ |
| 62 | typeof(index) _i = (index); \ |
| 63 | typeof(size) _s = (size); \ |
| 64 | unsigned long _mask = array_index_mask_nospec(_i, _s); \ |
| 65 | \ |
| 66 | BUILD_BUG_ON(sizeof(_i) > sizeof(long)); \ |
| 67 | BUILD_BUG_ON(sizeof(_s) > sizeof(long)); \ |
| 68 | \ |
| 69 | _i &= _mask; \ |
| 70 | _i; \ |
| 71 | }) |
| 72 | #endif /* _LINUX_NOSPEC_H */ |