mirror of
https://github.com/torvalds/linux.git
synced 2026-06-07 22:14:04 +02:00
0546200f0e
Adds xxhash kernel module with xxh32 and xxh64 hashes. xxhash is an
extremely fast non-cryptographic hash algorithm for checksumming.
The zstd compression and decompression modules added in the next patch
require xxhash. I extracted it out from zstd since it is useful on its
own. I copied the code from the upstream XXHash source repository and
translated it into kernel style. I ran benchmarks and tests in the kernel
and tests in userland.
I benchmarked xxhash as a special character device. I ran in four modes,
no-op, xxh32, xxh64, and crc32. The no-op mode simply copies the data to
kernel space and ignores it. The xxh32, xxh64, and crc32 modes compute
hashes on the copied data. I also ran it with four different buffer sizes.
The benchmark file is located in the upstream zstd source repository under
`contrib/linux-kernel/xxhash_test.c` [1].
I ran the benchmarks on a Ubuntu 14.04 VM with 2 cores and 4 GiB of RAM.
The VM is running on a MacBook Pro with a 3.1 GHz Intel Core i7 processor,
16 GB of RAM, and a SSD. I benchmarked using the file `filesystem.squashfs`
from `ubuntu-16.10-desktop-amd64.iso`, which is 1,536,217,088 B large.
Run the following commands for the benchmark:
modprobe xxhash_test
mknod xxhash_test c 245 0
time cp filesystem.squashfs xxhash_test
The time is reported by the time of the userland `cp`.
The GB/s is computed with
1,536,217,008 B / time(buffer size, hash)
which includes the time to copy from userland.
The Normalized GB/s is computed with
1,536,217,088 B / (time(buffer size, hash) - time(buffer size, none)).
| Buffer Size (B) | Hash | Time (s) | GB/s | Adjusted GB/s |
|-----------------|-------|----------|------|---------------|
| 1024 | none | 0.408 | 3.77 | - |
| 1024 | xxh32 | 0.649 | 2.37 | 6.37 |
| 1024 | xxh64 | 0.542 | 2.83 | 11.46 |
| 1024 | crc32 | 1.290 | 1.19 | 1.74 |
| 4096 | none | 0.380 | 4.04 | - |
| 4096 | xxh32 | 0.645 | 2.38 | 5.79 |
| 4096 | xxh64 | 0.500 | 3.07 | 12.80 |
| 4096 | crc32 | 1.168 | 1.32 | 1.95 |
| 8192 | none | 0.351 | 4.38 | - |
| 8192 | xxh32 | 0.614 | 2.50 | 5.84 |
| 8192 | xxh64 | 0.464 | 3.31 | 13.60 |
| 8192 | crc32 | 1.163 | 1.32 | 1.89 |
| 16384 | none | 0.346 | 4.43 | - |
| 16384 | xxh32 | 0.590 | 2.60 | 6.30 |
| 16384 | xxh64 | 0.466 | 3.30 | 12.80 |
| 16384 | crc32 | 1.183 | 1.30 | 1.84 |
Tested in userland using the test-suite in the zstd repo under
`contrib/linux-kernel/test/XXHashUserlandTest.cpp` [2] by mocking the
kernel functions. A line in each branch of every function in `xxhash.c`
was commented out to ensure that the test-suite fails. Additionally
tested while testing zstd and with SMHasher [3].
[1] https://phabricator.intern.facebook.com/P57526246
[2] https://github.com/facebook/zstd/blob/dev/contrib/linux-kernel/test/XXHashUserlandTest.cpp
[3] https://github.com/aappleby/smhasher
zstd source repository: https://github.com/facebook/zstd
XXHash source repository: https://github.com/cyan4973/xxhash
Change-Id: Ibb5ffee816e2593800c07263719bd1d4b802d8de
Signed-off-by: Nick Terrell <terrelln@fb.com>
Signed-off-by: Chris Mason <clm@fb.com>
Signed-off-by: Ziyuan Xu <xzy.xu@rock-chips.com>
(cherry-picked from
|
||
|---|---|---|
| .. | ||
| 842 | ||
| fonts | ||
| lz4 | ||
| lzo | ||
| mpi | ||
| raid6 | ||
| reed_solomon | ||
| xz | ||
| zlib_deflate | ||
| zlib_inflate | ||
| .gitignore | ||
| argv_split.c | ||
| asn1_decoder.c | ||
| assoc_array.c | ||
| atomic64_test.c | ||
| atomic64.c | ||
| audit.c | ||
| bcd.c | ||
| bch.c | ||
| bitmap.c | ||
| bitrev.c | ||
| bsearch.c | ||
| btree.c | ||
| bug.c | ||
| build_OID_registry | ||
| bust_spinlocks.c | ||
| check_signature.c | ||
| checksum.c | ||
| clz_ctz.c | ||
| clz_tab.c | ||
| cmdline.c | ||
| compat_audit.c | ||
| cordic.c | ||
| cpu_rmap.c | ||
| cpu-notifier-error-inject.c | ||
| cpumask.c | ||
| crc-ccitt.c | ||
| crc-itu-t.c | ||
| crc-t10dif.c | ||
| crc7.c | ||
| crc8.c | ||
| crc16.c | ||
| crc32.c | ||
| crc32defs.h | ||
| ctype.c | ||
| debug_info.c | ||
| debug_locks.c | ||
| debugobjects.c | ||
| dec_and_lock.c | ||
| decompress_bunzip2.c | ||
| decompress_inflate.c | ||
| decompress_unlz4.c | ||
| decompress_unlzma.c | ||
| decompress_unlzo.c | ||
| decompress_unxz.c | ||
| decompress.c | ||
| devres.c | ||
| digsig.c | ||
| div64.c | ||
| dma-debug.c | ||
| dump_stack.c | ||
| dynamic_debug.c | ||
| dynamic_queue_limits.c | ||
| earlycpio.c | ||
| extable.c | ||
| fault-inject.c | ||
| fdt_empty_tree.c | ||
| fdt_ro.c | ||
| fdt_rw.c | ||
| fdt_strerror.c | ||
| fdt_sw.c | ||
| fdt_wip.c | ||
| fdt.c | ||
| find_bit.c | ||
| flex_array.c | ||
| flex_proportions.c | ||
| gcd.c | ||
| gen_crc32table.c | ||
| genalloc.c | ||
| glob.c | ||
| halfmd4.c | ||
| hexdump.c | ||
| hweight.c | ||
| idr.c | ||
| inflate.c | ||
| int_sqrt.c | ||
| interval_tree_test.c | ||
| interval_tree.c | ||
| iomap_copy.c | ||
| iomap.c | ||
| iommu-common.c | ||
| iommu-helper.c | ||
| ioremap.c | ||
| iov_iter.c | ||
| irq_regs.c | ||
| is_single_threaded.c | ||
| jedec_ddr_data.c | ||
| kasprintf.c | ||
| Kconfig | ||
| Kconfig.debug | ||
| Kconfig.kasan | ||
| Kconfig.kgdb | ||
| Kconfig.kmemcheck | ||
| kfifo.c | ||
| klist.c | ||
| kobject_uevent.c | ||
| kobject.c | ||
| kstrtox.c | ||
| kstrtox.h | ||
| lcm.c | ||
| libcrc32c.c | ||
| list_debug.c | ||
| list_sort.c | ||
| llist.c | ||
| locking-selftest-hardirq.h | ||
| locking-selftest-mutex.h | ||
| locking-selftest-rlock-hardirq.h | ||
| locking-selftest-rlock-softirq.h | ||
| locking-selftest-rlock.h | ||
| locking-selftest-rsem.h | ||
| locking-selftest-softirq.h | ||
| locking-selftest-spin-hardirq.h | ||
| locking-selftest-spin-softirq.h | ||
| locking-selftest-spin.h | ||
| locking-selftest-wlock-hardirq.h | ||
| locking-selftest-wlock-softirq.h | ||
| locking-selftest-wlock.h | ||
| locking-selftest-wsem.h | ||
| locking-selftest.c | ||
| lockref.c | ||
| lru_cache.c | ||
| Makefile | ||
| md5.c | ||
| memory-notifier-error-inject.c | ||
| memweight.c | ||
| net_utils.c | ||
| nlattr.c | ||
| nmi_backtrace.c | ||
| notifier-error-inject.c | ||
| notifier-error-inject.h | ||
| of-reconfig-notifier-error-inject.c | ||
| oid_registry.c | ||
| once.c | ||
| parser.c | ||
| pci_iomap.c | ||
| percpu_counter.c | ||
| percpu_ida.c | ||
| percpu_test.c | ||
| percpu-refcount.c | ||
| plist.c | ||
| pm-notifier-error-inject.c | ||
| proportions.c | ||
| radix-tree.c | ||
| random32.c | ||
| ratelimit.c | ||
| rational.c | ||
| rbtree_test.c | ||
| rbtree.c | ||
| reciprocal_div.c | ||
| rhashtable.c | ||
| scatterlist.c | ||
| seq_buf.c | ||
| sg_split.c | ||
| sha1.c | ||
| show_mem.c | ||
| smp_processor_id.c | ||
| sort.c | ||
| stackdepot.c | ||
| stmp_device.c | ||
| string_helpers.c | ||
| string.c | ||
| strncpy_from_user.c | ||
| strnlen_user.c | ||
| swiotlb.c | ||
| syscall.c | ||
| test_bpf.c | ||
| test_firmware.c | ||
| test_kasan.c | ||
| test_module.c | ||
| test_printf.c | ||
| test_rhashtable.c | ||
| test_static_key_base.c | ||
| test_static_keys.c | ||
| test_user_copy.c | ||
| test-hexdump.c | ||
| test-kstrtox.c | ||
| test-string_helpers.c | ||
| textsearch.c | ||
| timerqueue.c | ||
| ts_bm.c | ||
| ts_fsm.c | ||
| ts_kmp.c | ||
| ucs2_string.c | ||
| usercopy.c | ||
| uuid.c | ||
| vsprintf.c | ||
| xxhash.c | ||