/* vim:set ts=2 sw=2 sts=2 et: */ /** * \author Marcus Holland-Moritz (github@mhxnet.de) * \copyright Copyright (c) Marcus Holland-Moritz * * This file is part of dwarfs. * * dwarfs is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation, either version 3 of the License, or * (at your option) any later version. * * dwarfs is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with dwarfs. If not, see . */ #include #include #include #include "dwarfs/block_compressor.h" #include "dwarfs/entry.h" #include "dwarfs/file_stat.h" #include "dwarfs/filesystem_v2.h" #include "dwarfs/filesystem_writer.h" #include "dwarfs/iovec_read_buf.h" #include "dwarfs/logger.h" #include "dwarfs/options.h" #include "dwarfs/progress.h" #include "dwarfs/scanner.h" #include "dwarfs/segmenter_factory.h" #include "dwarfs/string_table.h" #include "dwarfs/vfs_stat.h" #include "dwarfs/worker_group.h" #include "mmap_mock.h" #include "test_helpers.h" #include "test_logger.h" #include "test_strings.h" #include "dwarfs/gen-cpp2/metadata_layouts.h" namespace { using namespace dwarfs; void PackParams(::benchmark::internal::Benchmark* b) { for (auto pack_directories : {false, true}) { for (auto plain_tables : {false, true}) { if (plain_tables) { b->Args({pack_directories, true, false, false}); } else { for (auto pack_strings : {false, true}) { for (auto pack_strings_index : {false, true}) { b->Args( {pack_directories, false, pack_strings, pack_strings_index}); } } } } } } void PackParamsStrings(::benchmark::internal::Benchmark* b) { for (auto plain_tables : {false, true}) { if (plain_tables) { b->Args({true, true, false, false}); } else { for (auto pack_strings : {false, true}) { for (auto pack_strings_index : {false, true}) { b->Args({true, false, pack_strings, pack_strings_index}); } } } } } void PackParamsNone(::benchmark::internal::Benchmark* b) { b->Args({true, false, true, true}); } void PackParamsDirs(::benchmark::internal::Benchmark* b) { for (auto pack_directories : {false, true}) { b->Args({pack_directories, false, true, true}); } } std::string make_filesystem(::benchmark::State const& state) { segmenter_factory::config cfg; scanner_options options; cfg.blockhash_window_size.set_default(12); cfg.window_increment_shift.set_default(1); cfg.max_active_blocks.set_default(1); cfg.bloom_filter_size.set_default(4); cfg.block_size_bits = 12; options.with_devices = true; options.with_specials = true; options.keep_all_times = false; options.pack_chunk_table = true; options.pack_directories = state.range(0); options.pack_shared_files_table = true; options.pack_names = state.range(2); options.pack_names_index = state.range(3); options.pack_symlinks = state.range(2); options.pack_symlinks_index = state.range(3); options.force_pack_string_tables = true; options.plain_names_table = state.range(1); options.plain_symlinks_table = state.range(1); test::test_logger lgr; auto os = test::os_access_mock::create_test_instance(); worker_group wg(lgr, *os, "writer", 4); progress prog([](const progress&, bool) {}, 1000); auto sf = std::make_shared(lgr, prog, cfg); scanner s(lgr, wg, sf, entry_factory::create(), os, std::make_shared(), options); std::ostringstream oss; block_compressor bc("null"); filesystem_writer fsw(oss, lgr, wg, prog, bc, bc, bc); fsw.add_default_compressor(bc); s.scan(fsw, "", prog); return oss.str(); } template auto make_frozen_string_table(T const& strings, string_table::pack_options const& options) { using namespace apache::thrift::frozen; std::string tmp; auto tbl = string_table::pack(strings, options); freezeToString(tbl, tmp); return mapFrozen(std::move(tmp)); } auto make_frozen_legacy_string_table(std::vector&& strings) { using namespace apache::thrift::frozen; std::string tmp; freezeToString(strings, tmp); return mapFrozen>(std::move(tmp)); } void frozen_legacy_string_table_lookup(::benchmark::State& state) { auto data = make_frozen_legacy_string_table(test::test_string_vector()); string_table table(data); int i = 0; std::string str; for (auto _ : state) { ::benchmark::DoNotOptimize(str = table[i++ % test::NUM_STRINGS]); } } void frozen_string_table_lookup(::benchmark::State& state) { auto data = make_frozen_string_table( test::test_strings, string_table::pack_options(state.range(0), state.range(1), true)); test::test_logger lgr; string_table table(lgr, "bench", data); int i = 0; std::string str; for (auto _ : state) { ::benchmark::DoNotOptimize(str = table[i++ % test::NUM_STRINGS]); } } void dwarfs_initialize(::benchmark::State& state) { auto image = make_filesystem(state); test::test_logger lgr; test::os_access_mock os; auto mm = std::make_shared(image); filesystem_options opts; opts.block_cache.max_bytes = 1 << 20; opts.metadata.enable_nlink = true; for (auto _ : state) { auto fs = filesystem_v2(lgr, os, mm, opts); ::benchmark::DoNotOptimize(fs); } } class filesystem : public ::benchmark::Fixture { public: static constexpr size_t NUM_ENTRIES = 8; void SetUp(::benchmark::State const& state) { image = make_filesystem(state); mm = std::make_shared(image); filesystem_options opts; opts.block_cache.max_bytes = 1 << 20; opts.metadata.enable_nlink = true; fs = std::make_unique(lgr, os, mm, opts); entries.reserve(NUM_ENTRIES); for (int i = 0; entries.size() < NUM_ENTRIES; ++i) { if (auto e = fs->find(i)) { entries.emplace_back(*e); } } } void TearDown(::benchmark::State const&) { entries.clear(); image.clear(); mm.reset(); fs.reset(); } void read_bench(::benchmark::State& state, const char* file) { auto iv = fs->find(file); file_stat st; fs->getattr(*iv, &st); auto i = fs->open(*iv); std::string buf; buf.resize(st.size); for (auto _ : state) { auto r = fs->read(i, buf.data(), st.size); ::benchmark::DoNotOptimize(r); } } void readv_bench(::benchmark::State& state, char const* file) { auto iv = fs->find(file); file_stat st; fs->getattr(*iv, &st); auto i = fs->open(*iv); for (auto _ : state) { iovec_read_buf buf; auto r = fs->readv(i, buf, st.size); ::benchmark::DoNotOptimize(r); } } void readv_future_bench(::benchmark::State& state, char const* file) { auto iv = fs->find(file); file_stat st; fs->getattr(*iv, &st); auto i = fs->open(*iv); for (auto _ : state) { auto x = fs->readv(i, st.size); for (auto& f : *x) { auto r = f.get().size(); ::benchmark::DoNotOptimize(r); } } } template void getattr_bench(::benchmark::State& state, std::array const& paths) { int i = 0; std::vector ent; ent.reserve(paths.size()); for (auto const& p : paths) { ent.emplace_back(*fs->find(p.data())); } for (auto _ : state) { file_stat buf; auto r = fs->getattr(ent[i++ % N], &buf); ::benchmark::DoNotOptimize(r); } } std::unique_ptr fs; std::vector entries; private: test::test_logger lgr; test::os_access_mock os; std::string image; std::shared_ptr mm; }; BENCHMARK_DEFINE_F(filesystem, find_path)(::benchmark::State& state) { std::array paths{{ "/test.pl", "/somelink", "/baz.pl", "/ipsum.txt", "/somedir/ipsum.py", "/somedir/bad", "/somedir/null", "/somedir/zero", }}; int i = 0; for (auto _ : state) { auto r = fs->find(paths[i++ % paths.size()]); ::benchmark::DoNotOptimize(r); } } BENCHMARK_DEFINE_F(filesystem, find_inode)(::benchmark::State& state) { int i = 0; for (auto _ : state) { auto r = fs->find(i++ % 8); ::benchmark::DoNotOptimize(r); } } BENCHMARK_DEFINE_F(filesystem, find_inode_name)(::benchmark::State& state) { std::array names{{ "ipsum.py", "bad", "null", "zero", }}; auto base = fs->find("/somedir"); int i = 0; for (auto _ : state) { auto r = fs->find(base->inode_num(), names[i++ % names.size()]); ::benchmark::DoNotOptimize(r); } } BENCHMARK_DEFINE_F(filesystem, getattr_dir)(::benchmark::State& state) { std::array paths{{"/", "/somedir"}}; getattr_bench(state, paths); } BENCHMARK_DEFINE_F(filesystem, getattr_file)(::benchmark::State& state) { std::array paths{ {"/foo.pl", "/bar.pl", "/baz.pl", "/somedir/ipsum.py"}}; getattr_bench(state, paths); } BENCHMARK_DEFINE_F(filesystem, getattr_file_large)(::benchmark::State& state) { std::array paths{{"/ipsum.txt"}}; getattr_bench(state, paths); } BENCHMARK_DEFINE_F(filesystem, getattr_link)(::benchmark::State& state) { std::array paths{{"/somelink", "/somedir/bad"}}; getattr_bench(state, paths); } BENCHMARK_DEFINE_F(filesystem, getattr_dev)(::benchmark::State& state) { std::array paths{{"/somedir/null", "/somedir/zero"}}; getattr_bench(state, paths); } BENCHMARK_DEFINE_F(filesystem, access_F_OK)(::benchmark::State& state) { int i = 0; for (auto _ : state) { auto r = fs->access(entries[i++ % NUM_ENTRIES], F_OK, 1000, 100); ::benchmark::DoNotOptimize(r); } } BENCHMARK_DEFINE_F(filesystem, access_R_OK)(::benchmark::State& state) { int i = 0; for (auto _ : state) { auto r = fs->access(entries[i++ % NUM_ENTRIES], R_OK, 1000, 100); ::benchmark::DoNotOptimize(r); } } BENCHMARK_DEFINE_F(filesystem, opendir)(::benchmark::State& state) { auto iv = fs->find("/somedir"); for (auto _ : state) { auto r = fs->opendir(*iv); ::benchmark::DoNotOptimize(r); } } BENCHMARK_DEFINE_F(filesystem, dirsize)(::benchmark::State& state) { auto iv = fs->find("/somedir"); auto dv = fs->opendir(*iv); for (auto _ : state) { auto r = fs->dirsize(*dv); ::benchmark::DoNotOptimize(r); } } BENCHMARK_DEFINE_F(filesystem, readdir)(::benchmark::State& state) { auto iv = fs->find("/"); auto dv = fs->opendir(*iv); auto const num = fs->dirsize(*dv); size_t i = 0; for (auto _ : state) { auto r = fs->readdir(*dv, i % num); ::benchmark::DoNotOptimize(r); } } BENCHMARK_DEFINE_F(filesystem, readlink)(::benchmark::State& state) { auto iv = fs->find("/somelink"); for (auto _ : state) { auto r = fs->readlink(*iv); ::benchmark::DoNotOptimize(r); } } BENCHMARK_DEFINE_F(filesystem, statvfs)(::benchmark::State& state) { for (auto _ : state) { vfs_stat buf; auto r = fs->statvfs(&buf); ::benchmark::DoNotOptimize(r); } } BENCHMARK_DEFINE_F(filesystem, open)(::benchmark::State& state) { int i = 0; for (auto _ : state) { auto r = fs->open(entries[i++ % NUM_ENTRIES]); ::benchmark::DoNotOptimize(r); } } BENCHMARK_DEFINE_F(filesystem, read_small)(::benchmark::State& state) { read_bench(state, "/somedir/ipsum.py"); } BENCHMARK_DEFINE_F(filesystem, read_large)(::benchmark::State& state) { read_bench(state, "/ipsum.txt"); } BENCHMARK_DEFINE_F(filesystem, readv_small)(::benchmark::State& state) { readv_bench(state, "/somedir/ipsum.py"); } BENCHMARK_DEFINE_F(filesystem, readv_large)(::benchmark::State& state) { readv_bench(state, "/ipsum.txt"); } BENCHMARK_DEFINE_F(filesystem, readv_future_small)(::benchmark::State& state) { readv_future_bench(state, "/somedir/ipsum.py"); } BENCHMARK_DEFINE_F(filesystem, readv_future_large)(::benchmark::State& state) { readv_future_bench(state, "/ipsum.txt"); } } // namespace BENCHMARK(frozen_legacy_string_table_lookup); BENCHMARK(frozen_string_table_lookup) ->Args({false, false}) ->Args({false, true}) ->Args({true, false}) ->Args({true, true}); BENCHMARK(dwarfs_initialize)->Apply(PackParams); BENCHMARK_REGISTER_F(filesystem, find_inode)->Apply(PackParams); BENCHMARK_REGISTER_F(filesystem, find_inode_name)->Apply(PackParams); BENCHMARK_REGISTER_F(filesystem, find_path)->Apply(PackParams); BENCHMARK_REGISTER_F(filesystem, getattr_dir)->Apply(PackParamsNone); BENCHMARK_REGISTER_F(filesystem, getattr_link)->Apply(PackParamsNone); BENCHMARK_REGISTER_F(filesystem, getattr_file)->Apply(PackParamsNone); BENCHMARK_REGISTER_F(filesystem, getattr_file_large)->Apply(PackParamsNone); BENCHMARK_REGISTER_F(filesystem, getattr_dev)->Apply(PackParamsNone); BENCHMARK_REGISTER_F(filesystem, access_F_OK)->Apply(PackParamsNone); BENCHMARK_REGISTER_F(filesystem, access_R_OK)->Apply(PackParamsNone); BENCHMARK_REGISTER_F(filesystem, opendir)->Apply(PackParamsNone); BENCHMARK_REGISTER_F(filesystem, dirsize)->Apply(PackParamsDirs); BENCHMARK_REGISTER_F(filesystem, readdir)->Apply(PackParams); BENCHMARK_REGISTER_F(filesystem, readlink)->Apply(PackParamsStrings); BENCHMARK_REGISTER_F(filesystem, statvfs)->Apply(PackParamsNone); BENCHMARK_REGISTER_F(filesystem, open)->Apply(PackParamsNone); BENCHMARK_REGISTER_F(filesystem, read_small)->Apply(PackParamsNone); BENCHMARK_REGISTER_F(filesystem, read_large)->Apply(PackParamsNone); BENCHMARK_REGISTER_F(filesystem, readv_small)->Apply(PackParamsNone); BENCHMARK_REGISTER_F(filesystem, readv_large)->Apply(PackParamsNone); BENCHMARK_REGISTER_F(filesystem, readv_future_small)->Apply(PackParamsNone); BENCHMARK_REGISTER_F(filesystem, readv_future_large)->Apply(PackParamsNone); BENCHMARK_MAIN();