2017-11-05 21:59:12.467593 7f2338baee00  0 ceph version 12.2.1 (3e7492b9ada8bdc9a5cd0feafd42fbca27f9c38e) luminous (stable), process (unknown), pid 15193
2017-11-05 21:59:12.468352 7f2338baee00  0 pidfile_write: ignore empty --pid-file
2017-11-05 21:59:12.477930 7f2338baee00  0 load: jerasure load: lrc load: isa
2017-11-05 21:59:12.478102 7f2338baee00  0  set rocksdb option compression = kNoCompression
2017-11-05 21:59:12.478114 7f2338baee00  0  set rocksdb option write_buffer_size = 33554432
2017-11-05 21:59:12.478145 7f2338baee00  0  set rocksdb option compression = kNoCompression
2017-11-05 21:59:12.478152 7f2338baee00  0  set rocksdb option write_buffer_size = 33554432
2017-11-05 21:59:12.478341 7f2338baee00  4 rocksdb: RocksDB version: 5.4.0

2017-11-05 21:59:12.478353 7f2338baee00  4 rocksdb: Git sha rocksdb_build_git_sha:@0@
2017-11-05 21:59:12.478355 7f2338baee00  4 rocksdb: Compile date Sep 26 2017
2017-11-05 21:59:12.478361 7f2338baee00  4 rocksdb: DB SUMMARY

2017-11-05 21:59:12.478409 7f2338baee00  4 rocksdb: CURRENT file:  CURRENT

2017-11-05 21:59:12.478415 7f2338baee00  4 rocksdb: IDENTITY file:  IDENTITY

2017-11-05 21:59:12.478419 7f2338baee00  4 rocksdb: MANIFEST file:  MANIFEST-000057 size: 216 Bytes

2017-11-05 21:59:12.478422 7f2338baee00  4 rocksdb: SST files in /var/lib/ceph/mon/ceph-ceph03/store.db dir, Total Num: 3, files: 000051.sst 000053.sst 000056.sst

2017-11-05 21:59:12.478424 7f2338baee00  4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-ceph03/store.db: 000058.log size: 170108 ;

2017-11-05 21:59:12.478426 7f2338baee00  4 rocksdb:                         Options.error_if_exists: 0
2017-11-05 21:59:12.478428 7f2338baee00  4 rocksdb:                       Options.create_if_missing: 0
2017-11-05 21:59:12.478429 7f2338baee00  4 rocksdb:                         Options.paranoid_checks: 1
2017-11-05 21:59:12.478435 7f2338baee00  4 rocksdb:                                     Options.env: 0x7f2341cdb2e0
2017-11-05 21:59:12.478437 7f2338baee00  4 rocksdb:                                Options.info_log: 0x7f2342bab540
2017-11-05 21:59:12.478438 7f2338baee00  4 rocksdb:                          Options.max_open_files: -1
2017-11-05 21:59:12.478439 7f2338baee00  4 rocksdb:                Options.max_file_opening_threads: 16
2017-11-05 21:59:12.478445 7f2338baee00  4 rocksdb:                               Options.use_fsync: 0
2017-11-05 21:59:12.478446 7f2338baee00  4 rocksdb:                       Options.max_log_file_size: 0
2017-11-05 21:59:12.478447 7f2338baee00  4 rocksdb:                  Options.max_manifest_file_size: 18446744073709551615
2017-11-05 21:59:12.478448 7f2338baee00  4 rocksdb:                   Options.log_file_time_to_roll: 0
2017-11-05 21:59:12.478449 7f2338baee00  4 rocksdb:                       Options.keep_log_file_num: 1000
2017-11-05 21:59:12.478450 7f2338baee00  4 rocksdb:                    Options.recycle_log_file_num: 0
2017-11-05 21:59:12.478451 7f2338baee00  4 rocksdb:                         Options.allow_fallocate: 1
2017-11-05 21:59:12.478452 7f2338baee00  4 rocksdb:                        Options.allow_mmap_reads: 0
2017-11-05 21:59:12.478453 7f2338baee00  4 rocksdb:                       Options.allow_mmap_writes: 0
2017-11-05 21:59:12.478454 7f2338baee00  4 rocksdb:                        Options.use_direct_reads: 0
2017-11-05 21:59:12.478455 7f2338baee00  4 rocksdb:                        Options.use_direct_io_for_flush_and_compaction: 0
2017-11-05 21:59:12.478456 7f2338baee00  4 rocksdb:          Options.create_missing_column_families: 0
2017-11-05 21:59:12.478456 7f2338baee00  4 rocksdb:                              Options.db_log_dir:
2017-11-05 21:59:12.478457 7f2338baee00  4 rocksdb:                                 Options.wal_dir: /var/lib/ceph/mon/ceph-ceph03/store.db
2017-11-05 21:59:12.478459 7f2338baee00  4 rocksdb:                Options.table_cache_numshardbits: 6
2017-11-05 21:59:12.478459 7f2338baee00  4 rocksdb:                      Options.max_subcompactions: 1
2017-11-05 21:59:12.478469 7f2338baee00  4 rocksdb:                  Options.max_background_flushes: 1
2017-11-05 21:59:12.478470 7f2338baee00  4 rocksdb:                         Options.WAL_ttl_seconds: 0
2017-11-05 21:59:12.478470 7f2338baee00  4 rocksdb:                       Options.WAL_size_limit_MB: 0
2017-11-05 21:59:12.478471 7f2338baee00  4 rocksdb:             Options.manifest_preallocation_size: 4194304
2017-11-05 21:59:12.478472 7f2338baee00  4 rocksdb:                     Options.is_fd_close_on_exec: 1
2017-11-05 21:59:12.478473 7f2338baee00  4 rocksdb:                   Options.advise_random_on_open: 1
2017-11-05 21:59:12.478474 7f2338baee00  4 rocksdb:                    Options.db_write_buffer_size: 0
2017-11-05 21:59:12.478475 7f2338baee00  4 rocksdb:         Options.access_hint_on_compaction_start: 1
2017-11-05 21:59:12.478476 7f2338baee00  4 rocksdb:  Options.new_table_reader_for_compaction_inputs: 0
2017-11-05 21:59:12.478476 7f2338baee00  4 rocksdb:               Options.compaction_readahead_size: 0
2017-11-05 21:59:12.478477 7f2338baee00  4 rocksdb:           Options.random_access_max_buffer_size: 1048576
2017-11-05 21:59:12.478478 7f2338baee00  4 rocksdb:           Options.writable_file_max_buffer_size: 1048576
2017-11-05 21:59:12.478479 7f2338baee00  4 rocksdb:                      Options.use_adaptive_mutex: 0
2017-11-05 21:59:12.478480 7f2338baee00  4 rocksdb:                            Options.rate_limiter: (nil)
2017-11-05 21:59:12.478481 7f2338baee00  4 rocksdb:     Options.sst_file_manager.rate_bytes_per_sec: 0
2017-11-05 21:59:12.478482 7f2338baee00  4 rocksdb:                          Options.bytes_per_sync: 0
2017-11-05 21:59:12.478483 7f2338baee00  4 rocksdb:                      Options.wal_bytes_per_sync: 0
2017-11-05 21:59:12.478484 7f2338baee00  4 rocksdb:                       Options.wal_recovery_mode: 2
2017-11-05 21:59:12.478485 7f2338baee00  4 rocksdb:                  Options.enable_thread_tracking: 0
2017-11-05 21:59:12.478486 7f2338baee00  4 rocksdb:         Options.allow_concurrent_memtable_write: 1
2017-11-05 21:59:12.478487 7f2338baee00  4 rocksdb:      Options.enable_write_thread_adaptive_yield: 1
2017-11-05 21:59:12.478488 7f2338baee00  4 rocksdb:             Options.write_thread_max_yield_usec: 100
2017-11-05 21:59:12.478488 7f2338baee00  4 rocksdb:            Options.write_thread_slow_yield_usec: 3
2017-11-05 21:59:12.478489 7f2338baee00  4 rocksdb:                               Options.row_cache: None
2017-11-05 21:59:12.478495 7f2338baee00  4 rocksdb:                              Options.wal_filter: None
2017-11-05 21:59:12.478496 7f2338baee00  4 rocksdb:             Options.avoid_flush_during_recovery: 0
2017-11-05 21:59:12.478498 7f2338baee00  4 rocksdb:             Options.base_background_compactions: 1
2017-11-05 21:59:12.478498 7f2338baee00  4 rocksdb:             Options.max_background_compactions: 1
2017-11-05 21:59:12.478499 7f2338baee00  4 rocksdb:             Options.avoid_flush_during_shutdown: 0
2017-11-05 21:59:12.478500 7f2338baee00  4 rocksdb:             Options.delayed_write_rate : 16777216
2017-11-05 21:59:12.478501 7f2338baee00  4 rocksdb:             Options.max_total_wal_size: 0
2017-11-05 21:59:12.478502 7f2338baee00  4 rocksdb:             Options.delete_obsolete_files_period_micros: 21600000000
2017-11-05 21:59:12.478504 7f2338baee00  4 rocksdb:                   Options.stats_dump_period_sec: 600
2017-11-05 21:59:12.478505 7f2338baee00  4 rocksdb: Compression algorithms supported:
2017-11-05 21:59:12.478505 7f2338baee00  4 rocksdb:     Snappy supported: 0
2017-11-05 21:59:12.478506 7f2338baee00  4 rocksdb:     Zlib supported: 0
2017-11-05 21:59:12.478507 7f2338baee00  4 rocksdb:     Bzip supported: 0
2017-11-05 21:59:12.478508 7f2338baee00  4 rocksdb:     LZ4 supported: 0
2017-11-05 21:59:12.478509 7f2338baee00  4 rocksdb:     ZSTD supported: 0
2017-11-05 21:59:12.478510 7f2338baee00  4 rocksdb: Fast CRC32 supported: 0
2017-11-05 21:59:12.478678 7f2338baee00  4 rocksdb: [/build/ceph-12.2.1/src/rocksdb/db/version_set.cc:2609] Recovering from manifest file: MANIFEST-000057

2017-11-05 21:59:12.478777 7f2338baee00  4 rocksdb: [/build/ceph-12.2.1/src/rocksdb/db/column_family.cc:407] --------------- Options for column family [default]:

2017-11-05 21:59:12.478786 7f2338baee00  4 rocksdb:               Options.comparator: leveldb.BytewiseComparator
2017-11-05 21:59:12.478788 7f2338baee00  4 rocksdb:           Options.merge_operator:
2017-11-05 21:59:12.478789 7f2338baee00  4 rocksdb:        Options.compaction_filter: None
2017-11-05 21:59:12.478792 7f2338baee00  4 rocksdb:        Options.compaction_filter_factory: None
2017-11-05 21:59:12.478793 7f2338baee00  4 rocksdb:         Options.memtable_factory: SkipListFactory
2017-11-05 21:59:12.478800 7f2338baee00  4 rocksdb:            Options.table_factory: BlockBasedTable
2017-11-05 21:59:12.478827 7f2338baee00  4 rocksdb:            table_factory options:   flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x7f23428e4170)
  cache_index_and_filter_blocks: 1
  cache_index_and_filter_blocks_with_high_priority: 1
  pin_l0_filter_and_index_blocks_in_cache: 1
  index_type: 0
  hash_index_allow_collision: 1
  checksum: 1
  no_block_cache: 0
  block_cache: 0x7f23429a5598
  block_cache_name: LRUCache
  block_cache_options:
    capacity : 134217728
    num_shard_bits : 4
    strict_capacity_limit : 0
    high_pri_pool_ratio: 0.000
  block_cache_compressed: (nil)
  persistent_cache: (nil)
  block_size: 4096
  block_size_deviation: 10
  block_restart_interval: 16
  index_block_restart_interval: 1
  filter_policy: rocksdb.BuiltinBloomFilter
  whole_key_filtering: 1
  format_version: 2

2017-11-05 21:59:12.478832 7f2338baee00  4 rocksdb:        Options.write_buffer_size: 33554432
2017-11-05 21:59:12.478834 7f2338baee00  4 rocksdb:  Options.max_write_buffer_number: 2
2017-11-05 21:59:12.478835 7f2338baee00  4 rocksdb:          Options.compression: NoCompression
2017-11-05 21:59:12.478837 7f2338baee00  4 rocksdb:                  Options.bottommost_compression: Disabled
2017-11-05 21:59:12.478838 7f2338baee00  4 rocksdb:       Options.prefix_extractor: nullptr
2017-11-05 21:59:12.478839 7f2338baee00  4 rocksdb:   Options.memtable_insert_with_hint_prefix_extractor: nullptr
2017-11-05 21:59:12.478840 7f2338baee00  4 rocksdb:             Options.num_levels: 7
2017-11-05 21:59:12.478841 7f2338baee00  4 rocksdb:        Options.min_write_buffer_number_to_merge: 1
2017-11-05 21:59:12.478842 7f2338baee00  4 rocksdb:     Options.max_write_buffer_number_to_maintain: 0
2017-11-05 21:59:12.478843 7f2338baee00  4 rocksdb:            Options.compression_opts.window_bits: -14
2017-11-05 21:59:12.478843 7f2338baee00  4 rocksdb:                  Options.compression_opts.level: -1
2017-11-05 21:59:12.478844 7f2338baee00  4 rocksdb:               Options.compression_opts.strategy: 0
2017-11-05 21:59:12.478845 7f2338baee00  4 rocksdb:         Options.compression_opts.max_dict_bytes: 0
2017-11-05 21:59:12.478846 7f2338baee00  4 rocksdb:      Options.level0_file_num_compaction_trigger: 4
2017-11-05 21:59:12.478847 7f2338baee00  4 rocksdb:          Options.level0_slowdown_writes_trigger: 20
2017-11-05 21:59:12.478848 7f2338baee00  4 rocksdb:              Options.level0_stop_writes_trigger: 36
2017-11-05 21:59:12.478849 7f2338baee00  4 rocksdb:                   Options.target_file_size_base: 67108864
2017-11-05 21:59:12.478850 7f2338baee00  4 rocksdb:             Options.target_file_size_multiplier: 1
2017-11-05 21:59:12.478851 7f2338baee00  4 rocksdb:                Options.max_bytes_for_level_base: 268435456
2017-11-05 21:59:12.478852 7f2338baee00  4 rocksdb: Options.level_compaction_dynamic_level_bytes: 0
2017-11-05 21:59:12.478853 7f2338baee00  4 rocksdb:          Options.max_bytes_for_level_multiplier: 10.000000
2017-11-05 21:59:12.478856 7f2338baee00  4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1
2017-11-05 21:59:12.478858 7f2338baee00  4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1
2017-11-05 21:59:12.478859 7f2338baee00  4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1
2017-11-05 21:59:12.478860 7f2338baee00  4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1
2017-11-05 21:59:12.478861 7f2338baee00  4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1
2017-11-05 21:59:12.478862 7f2338baee00  4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1
2017-11-05 21:59:12.478863 7f2338baee00  4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1
2017-11-05 21:59:12.478864 7f2338baee00  4 rocksdb:       Options.max_sequential_skip_in_iterations: 8
2017-11-05 21:59:12.478865 7f2338baee00  4 rocksdb:                    Options.max_compaction_bytes: 1677721600
2017-11-05 21:59:12.478866 7f2338baee00  4 rocksdb:                        Options.arena_block_size: 4194304
2017-11-05 21:59:12.478867 7f2338baee00  4 rocksdb:   Options.soft_pending_compaction_bytes_limit: 68719476736
2017-11-05 21:59:12.478868 7f2338baee00  4 rocksdb:   Options.hard_pending_compaction_bytes_limit: 274877906944
2017-11-05 21:59:12.478877 7f2338baee00  4 rocksdb:       Options.rate_limit_delay_max_milliseconds: 100
2017-11-05 21:59:12.478878 7f2338baee00  4 rocksdb:                Options.disable_auto_compactions: 0
2017-11-05 21:59:12.478885 7f2338baee00  4 rocksdb:                         Options.compaction_style: kCompactionStyleLevel
2017-11-05 21:59:12.478891 7f2338baee00  4 rocksdb:                           Options.compaction_pri: kByCompensatedSize
2017-11-05 21:59:12.478903 7f2338baee00  4 rocksdb:  Options.compaction_options_universal.size_ratio: 1
2017-11-05 21:59:12.478904 7f2338baee00  4 rocksdb: Options.compaction_options_universal.min_merge_width: 2
2017-11-05 21:59:12.478905 7f2338baee00  4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295
2017-11-05 21:59:12.478906 7f2338baee00  4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200
2017-11-05 21:59:12.478907 7f2338baee00  4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1
2017-11-05 21:59:12.478908 7f2338baee00  4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824
2017-11-05 21:59:12.478909 7f2338baee00  4 rocksdb:                   Options.table_properties_collectors:
2017-11-05 21:59:12.478910 7f2338baee00  4 rocksdb:                   Options.inplace_update_support: 0
2017-11-05 21:59:12.478911 7f2338baee00  4 rocksdb:                 Options.inplace_update_num_locks: 10000
2017-11-05 21:59:12.478912 7f2338baee00  4 rocksdb:               Options.memtable_prefix_bloom_size_ratio: 0.000000
2017-11-05 21:59:12.478914 7f2338baee00  4 rocksdb:   Options.memtable_huge_page_size: 0
2017-11-05 21:59:12.478915 7f2338baee00  4 rocksdb:                           Options.bloom_locality: 0
2017-11-05 21:59:12.478916 7f2338baee00  4 rocksdb:                    Options.max_successive_merges: 0
2017-11-05 21:59:12.478917 7f2338baee00  4 rocksdb:                Options.optimize_filters_for_hits: 0
2017-11-05 21:59:12.478918 7f2338baee00  4 rocksdb:                Options.paranoid_file_checks: 0
2017-11-05 21:59:12.478919 7f2338baee00  4 rocksdb:                Options.force_consistency_checks: 0
2017-11-05 21:59:12.478920 7f2338baee00  4 rocksdb:                Options.report_bg_io_stats: 0
2017-11-05 21:59:12.479894 7f2338baee00  4 rocksdb: [/build/ceph-12.2.1/src/rocksdb/db/version_set.cc:2859] Recovered from manifest file:/var/lib/ceph/mon/ceph-ceph03/store.db/MANIFEST-000057 succeeded,manifest_file_number is 57, next_file_number is 59, last_sequence is 11766, log_number is 0,prev_log_number is 0,max_column_family is 0

2017-11-05 21:59:12.479908 7f2338baee00  4 rocksdb: [/build/ceph-12.2.1/src/rocksdb/db/version_set.cc:2867] Column family [default] (ID 0), log number is 56

2017-11-05 21:59:12.479982 7f2338baee00  4 rocksdb: EVENT_LOG_v1 {"time_micros": 1509915552479972, "job": 1, "event": "recovery_started", "log_files": [58]}
2017-11-05 21:59:12.479990 7f2338baee00  4 rocksdb: [/build/ceph-12.2.1/src/rocksdb/db/db_impl_open.cc:482] Recovering log #58 mode 2
2017-11-05 21:59:12.527095 7f2338baee00  4 rocksdb: EVENT_LOG_v1 {"time_micros": 1509915552527081, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 59, "file_size": 113806, "table_properties": {"data_size": 112012, "index_size": 500, "filter_size": 377, "raw_key_size": 901, "raw_average_key_size": 24, "raw_value_size": 110963, "raw_average_value_size": 2999, "num_data_blocks": 15, "num_entries": 37, "filter_policy_name": "rocksdb.BuiltinBloomFilter", "kDeletedKeys": "0", "kMergeOperands": "0"}}
2017-11-05 21:59:12.527135 7f2338baee00  4 rocksdb: [/build/ceph-12.2.1/src/rocksdb/db/version_set.cc:2395] Creating manifest 60

2017-11-05 21:59:12.617442 7f2338baee00  4 rocksdb: EVENT_LOG_v1 {"time_micros": 1509915552617437, "job": 1, "event": "recovery_finished"}
2017-11-05 21:59:12.685504 7f2338baee00  4 rocksdb: [/build/ceph-12.2.1/src/rocksdb/db/db_impl_open.cc:1063] DB pointer 0x7f2342ca0000
2017-11-05 21:59:12.686191 7f2338baee00 10 obtain_monmap
2017-11-05 21:59:12.686560 7f2338baee00 10 obtain_monmap read last committed monmap ver 14
2017-11-05 21:59:12.687317 7f2338baee00  0 starting mon.ceph03 rank 2 at public addr 10.10.1.23:6789/0 at bind addr 10.10.1.23:6789/0 mon_data /var/lib/ceph/mon/ceph-ceph03 fsid 726a42b3-ae4c-4d0b-82c5-0a9065711b5f
2017-11-05 21:59:12.687485 7f2338baee00  0 starting mon.ceph03 rank 2 at 10.10.1.23:6789/0 mon_data /var/lib/ceph/mon/ceph-ceph03 fsid 726a42b3-ae4c-4d0b-82c5-0a9065711b5f
2017-11-05 21:59:12.687515 7f2338baee00  5 adding auth protocol: cephx
2017-11-05 21:59:12.687522 7f2338baee00  5 adding auth protocol: cephx
2017-11-05 21:59:12.688266 7f2338baee00  1 mon.ceph03@-1(probing) e14 preinit fsid 726a42b3-ae4c-4d0b-82c5-0a9065711b5f
2017-11-05 21:59:12.688405 7f2338baee00 10 mon.ceph03@-1(probing) e14 check_fsid cluster_uuid contains '726a42b3-ae4c-4d0b-82c5-0a9065711b5f'
2017-11-05 21:59:12.688436 7f2338baee00 10 mon.ceph03@-1(probing) e14 features compat={},rocompat={},incompat={1=initial feature set (~v.18),3=single paxos with k/v store (v0.?),4=support erasure code pools,5=new-style osdmap encoding,6=support isa/lrc erasure code,7=support shec erasure code,8=support monmap features,9=luminous ondisk layout}
2017-11-05 21:59:12.688451 7f2338baee00 10 mon.ceph03@-1(probing) e14 calc_quorum_requirements required_features 153140804152475648
2017-11-05 21:59:12.688455 7f2338baee00 10 mon.ceph03@-1(probing) e14 required_features 153140804152475648
2017-11-05 21:59:12.688471 7f2338baee00 10 mon.ceph03@-1(probing) e14 has_ever_joined = 1
2017-11-05 21:59:12.688526 7f2338baee00 10 mon.ceph03@-1(probing) e14 sync_last_committed_floor 0
2017-11-05 21:59:12.688532 7f2338baee00 10 mon.ceph03@-1(probing) e14 init_paxos
2017-11-05 21:59:12.688613 7f2338baee00 10 mon.ceph03@-1(probing).paxos(paxos recovering c 112907080..112907717) init last_pn: 12300 accepted_pn: 13500 last_committed: 112907717 first_committed: 112907080
2017-11-05 21:59:12.688635 7f2338baee00 10 mon.ceph03@-1(probing).paxos(paxos recovering c 112907080..112907717) init
2017-11-05 21:59:12.688657 7f2338baee00  1 mon.ceph03@-1(probing).mds e0 Unable to load 'last_metadata'
2017-11-05 21:59:12.688699 7f2338baee00 10 mon.ceph03@-1(probing).health init
2017-11-05 21:59:12.688705 7f2338baee00 10 mon.ceph03@-1(probing) e14 refresh_from_paxos
2017-11-05 21:59:12.688716 7f2338baee00 10 mon.ceph03@-1(probing) e14 refresh_from_paxos no cluster_fingerprint
2017-11-05 21:59:12.688770 7f2338baee00  1 mon.ceph03@-1(probing).paxosservice(pgmap 56600067..56600783) refresh upgraded, format 0 -> 1
2017-11-05 21:59:12.688779 7f2338baee00  1 mon.ceph03@-1(probing).pg v0 on_upgrade discarding in-core PGMap
2017-11-05 21:59:12.688792 7f2338baee00 10 mon.ceph03@-1(probing).paxosservice(pgmap 56600067..56600783) refresh
2017-11-05 21:59:12.688803 7f2338baee00 10 mon.ceph03@-1(probing).pg v0 update_from_paxos deleted, clearing in-memory PGMap
2017-11-05 21:59:12.688856 7f2338baee00 10 mon.ceph03@-1(probing).paxosservice(mdsmap 1..1) refresh
2017-11-05 21:59:12.688863 7f2338baee00 10 mon.ceph03@-1(probing).mds e0 update_from_paxos version 1, my e 0
2017-11-05 21:59:12.688888 7f2338baee00 10 mon.ceph03@-1(probing).mds e0 update_from_paxos got 1
2017-11-05 21:59:12.688917 7f2338baee00  4 mon.ceph03@-1(probing).mds e1 new map
2017-11-05 21:59:12.688922 7f2338baee00  0 mon.ceph03@-1(probing).mds e1 print_map
e1
enable_multiple, ever_enabled_multiple: 0,0
compat: compat={},rocompat={},incompat={}
legacy client fscid: -1

No filesystems configured

2017-11-05 21:59:12.688937 7f2338baee00 10 mon.ceph03@-1(probing).mds e1 update_logger
2017-11-05 21:59:12.689075 7f2338baee00 10 mon.ceph03@-1(probing).paxosservice(osdmap 503..1105) refresh
2017-11-05 21:59:12.689089 7f2338baee00 15 mon.ceph03@-1(probing).osd e0 update_from_paxos paxos e 1105, my e 0
2017-11-05 21:59:12.689173 7f2338baee00  7 mon.ceph03@-1(probing).osd e0 update_from_paxos loading latest full map e1105
2017-11-05 21:59:12.689357 7f2338baee00  7 mon.ceph03@-1(probing).osd e1105 update_from_paxos loading creating_pgs last_scan_epoch 1104 with 0 pgs
2017-11-05 21:59:12.689368 7f2338baee00 10 mon.ceph03@-1(probing).osd e1105 update_from_paxos pgservice is mgrstat
2017-11-05 21:59:12.689372 7f2338baee00 10 mon.ceph03@-1(probing).osd e1105 check_osdmap_subs
2017-11-05 21:59:12.689382 7f2338baee00 10 mon.ceph03@-1(probing).osd e1105 share_map_with_random_osd no up osd on our session map
2017-11-05 21:59:12.689385 7f2338baee00 10 mon.ceph03@-1(probing).osd e1105 update_logger
2017-11-05 21:59:12.689390 7f2338baee00  0 mon.ceph03@-1(probing).osd e1105 crush map has features 1107558400, adjusting msgr requires
2017-11-05 21:59:12.689393 7f2338baee00  0 mon.ceph03@-1(probing).osd e1105 crush map has features 1107558400, adjusting msgr requires
2017-11-05 21:59:12.689398 7f2338baee00  0 mon.ceph03@-1(probing).osd e1105 crush map has features 720575941486854144, adjusting msgr requires
2017-11-05 21:59:12.689413 7f2338baee00  0 mon.ceph03@-1(probing).osd e1105 crush map has features 1107558400, adjusting msgr requires
2017-11-05 21:59:12.689455 7f2338baee00 20 queue 0x7f2342d4ae00 1 [0,512)
2017-11-05 21:59:12.689462 7f2338baee00 20 queue 0x7f2342d4ae00 2 [0,128)
2017-11-05 21:59:12.689464 7f2338baee00 20 queue 0x7f2342d4ae00 4 [0,128)
2017-11-05 21:59:12.689465 7f2338baee00 10 mon.ceph03@-1(probing).osd e1105 start_mapping started mapping job 0x7f2342d4ae00 at 2017-11-05 21:59:12.689416
2017-11-05 21:59:12.689587 7f2338baee00 10 mon.ceph03@-1(probing).paxosservice(logm 55596915..55597627) refresh
2017-11-05 21:59:12.689595 7f2338baee00 10 mon.ceph03@-1(probing).log v55597627 update_from_paxos
2017-11-05 21:59:12.689598 7f2338baee00 10 mon.ceph03@-1(probing).log v55597627 update_from_paxos version 55597627 summary v 0
2017-11-05 21:59:12.689608 7f2338baee00 10 mon.ceph03@-1(probing).log v55597627 update_from_paxos latest full 55597626
2017-11-05 21:59:12.689664 7f2338baee00  7 mon.ceph03@-1(probing).log v55597627 update_from_paxos loading summary e55597626
2017-11-05 21:59:12.689819 7f2338baee00  7 mon.ceph03@-1(probing).log v55597627 update_from_paxos loaded summary e55597626
2017-11-05 21:59:12.689856 7f2338baee00  7 mon.ceph03@-1(probing).log v55597627 update_from_paxos applying incremental log 55597627 2017-11-05 21:58:22.490661 mgr.ceph01mgr client.54805773 10.10.1.21:0/1619113657 1809 : audit [DBG] from='client.55026351 10.10.1.23:0/1865493862' entity='client.admin' cmd=[{"prefix": "osd pool stats", "target": ["mgr", ""]}]: dispatch
2017-11-05 21:59:12.689880 7f2338baee00 20 mon.ceph03@-1(probing).log v55597627 update_from_paxos logging for channel 'audit' to file '/var/log/ceph/ceph.audit.log'
2017-11-05 21:59:12.689891 7f2338baee00  7 mon.ceph03@-1(probing).log v55597627 update_from_paxos applying incremental log 55597627 2017-11-05 21:58:25.101829 mgr.ceph01mgr client.54805773 10.10.1.21:0/1619113657 1810 : audit [DBG] from='mon.0 10.10.1.21:0/24116' entity='mon.' cmd=[{"prefix": "osd pool stats"}]: dispatch
2017-11-05 21:59:12.689901 7f2338baee00 20 mon.ceph03@-1(probing).log v55597627 update_from_paxos logging for channel 'audit' to file '/var/log/ceph/ceph.audit.log'
2017-11-05 21:59:12.689908 7f2338baee00  7 mon.ceph03@-1(probing).log v55597627 update_from_paxos applying incremental log 55597627 2017-11-05 21:58:26.489612 mgr.ceph01mgr client.54805773 10.10.1.21:0/1619113657 1811 : audit [DBG] from='client.55006306 10.10.1.21:0/3517645621' entity='client.admin' cmd=[{"prefix": "osd pool stats", "target": ["mgr", ""], "format": "json"}]: dispatch
2017-11-05 21:59:12.689915 7f2338baee00 20 mon.ceph03@-1(probing).log v55597627 update_from_paxos logging for channel 'audit' to file '/var/log/ceph/ceph.audit.log'
2017-11-05 21:59:12.689920 7f2338baee00  7 mon.ceph03@-1(probing).log v55597627 update_from_paxos applying incremental log 55597627 2017-11-05 21:58:30.371628 mgr.ceph01mgr client.54805773 10.10.1.21:0/1619113657 1812 : audit [DBG] from='client.55026357 10.10.1.23:0/435814924' entity='client.admin' cmd=[{"prefix": "osd pool stats", "target": ["mgr", ""]}]: dispatch
2017-11-05 21:59:12.689926 7f2338baee00 20 mon.ceph03@-1(probing).log v55597627 update_from_paxos logging for channel 'audit' to file '/var/log/ceph/ceph.audit.log'
2017-11-05 21:59:12.689933 7f2338baee00 15 mon.ceph03@-1(probing).log v55597627 update_from_paxos logging for 0 channels
2017-11-05 21:59:12.689935 7f2338baee00 10 mon.ceph03@-1(probing).log v55597627 check_subs
2017-11-05 21:59:12.689968 7f2338baee00 10 mon.ceph03@-1(probing).paxosservice(monmap 1..14) refresh
2017-11-05 21:59:12.690012 7f2338baee00  1 mon.ceph03@-1(probing).paxosservice(auth 26252..26331) refresh upgraded, format 0 -> 2
2017-11-05 21:59:12.690019 7f2338baee00 10 mon.ceph03@-1(probing).paxosservice(auth 26252..26331) refresh
2017-11-05 21:59:12.690022 7f2338baee00 10 mon.ceph03@-1(probing).auth v26331 update_from_paxos
2017-11-05 21:59:12.690048 7f2338baee00 10 mon.ceph03@-1(probing).auth v26331 update_from_paxos version 26331 keys ver 0 latest 26321
2017-11-05 21:59:12.690080 7f2338baee00  7 mon.ceph03@-1(probing).auth v26331 update_from_paxos loading summary e 26321
2017-11-05 21:59:12.690086 7f2338baee00  7 mon.ceph03@-1(probing).auth v26331 update_from_paxos latest length 3603
2017-11-05 21:59:12.690788 7f2338baee00 10 mon.ceph03@-1(probing).auth v26331 update_from_paxos key server version 26321
2017-11-05 21:59:12.690813 7f2338baee00 20 mon.ceph03@-1(probing).auth v26331 update_from_paxos walking through version 26322 len 19
2017-11-05 21:59:12.690830 7f2338baee00 20 mon.ceph03@-1(probing).auth v26331 update_from_paxos walking through version 26323 len 19
2017-11-05 21:59:12.690845 7f2338baee00 20 mon.ceph03@-1(probing).auth v26331 update_from_paxos walking through version 26324 len 19
2017-11-05 21:59:12.690859 7f2338baee00 20 mon.ceph03@-1(probing).auth v26331 update_from_paxos walking through version 26325 len 19
2017-11-05 21:59:12.690869 7f2338baee00 20 mon.ceph03@-1(probing).auth v26331 update_from_paxos walking through version 26326 len 19
2017-11-05 21:59:12.690882 7f2338baee00 20 mon.ceph03@-1(probing).auth v26331 update_from_paxos walking through version 26327 len 19
2017-11-05 21:59:12.690893 7f2338baee00 20 mon.ceph03@-1(probing).auth v26331 update_from_paxos walking through version 26328 len 19
2017-11-05 21:59:12.690971 7f2338baee00 20 mon.ceph03@-1(probing).auth v26331 update_from_paxos walking through version 26329 len 19
2017-11-05 21:59:12.691020 7f2338baee00 20 mon.ceph03@-1(probing).auth v26331 update_from_paxos walking through version 26330 len 19
2017-11-05 21:59:12.691034 7f2338baee00 20 mon.ceph03@-1(probing).auth v26331 update_from_paxos walking through version 26331 len 19
2017-11-05 21:59:12.691041 7f2338baee00 10 mon.ceph03@-1(probing).auth v26331 update_from_paxos() last_allocated_id=55234096 max_global_id=55234096 format_version 2
2017-11-05 21:59:12.691095 7f2338baee00 10 mon.ceph03@-1(probing).paxosservice(mgr 1..394) refresh
2017-11-05 21:59:12.691102 7f2338baee00  4 mon.ceph03@-1(probing).mgr e0 loading version 394
2017-11-05 21:59:12.691140 7f2338baee00  4 mon.ceph03@-1(probing).mgr e394 active server: 10.10.1.21:6806/22556(54805773)
2017-11-05 21:59:12.691176 7f2338baee00  4 mon.ceph03@-1(probing).mgr e394 mkfs or daemon transitioned to available, loading commands
2017-11-05 21:59:12.691190 7f2338baee00 -1 mon.ceph03@-1(probing).mgr e394 Failed to load mgr commands: (2) No such file or directory
2017-11-05 21:59:12.691296 7f2338baee00 10 mon.ceph03@-1(probing).paxosservice(mgrstat 56609513..56609621) refresh
2017-11-05 21:59:12.691304 7f2338baee00 10 mon.ceph03@-1(probing).mgrstat  56609621
2017-11-05 21:59:12.691380 7f2338baee00 10 mon.ceph03@-1(probing).mgrstat update_from_paxos v56609621 service_map e1
2017-11-05 21:59:12.691388 7f2338baee00 10 mon.ceph03@-1(probing).mgrstat check_subs
2017-11-05 21:59:12.691390 7f2338baee00 20 mon.ceph03@-1(probing).mgrstat update_logger
2017-11-05 21:59:12.691425 7f2338baee00 10 mon.ceph03@-1(probing).paxosservice(health 1..22) refresh
2017-11-05 21:59:12.691432 7f2338baee00 10 mon.ceph03@-1(probing).health update_from_paxos
2017-11-05 21:59:12.691463 7f2338baee00 20 mon.ceph03@-1(probing).health dump:{
    "quorum_health": {},
    "leader_health": {}
}

2017-11-05 21:59:12.691479 7f2338baee00 10 mon.ceph03@-1(probing).paxosservice(pgmap 56600067..56600783) post_refresh
2017-11-05 21:59:12.691481 7f2338baee00 10 mon.ceph03@-1(probing).paxosservice(mdsmap 1..1) post_refresh
2017-11-05 21:59:12.691483 7f2338baee00 10 mon.ceph03@-1(probing).paxosservice(osdmap 503..1105) post_refresh
2017-11-05 21:59:12.691485 7f2338baee00 10 mon.ceph03@-1(probing).paxosservice(logm 55596915..55597627) post_refresh
2017-11-05 21:59:12.691486 7f2338baee00 10 mon.ceph03@-1(probing).paxosservice(monmap 1..14) post_refresh
2017-11-05 21:59:12.691488 7f2338baee00 10 mon.ceph03@-1(probing).paxosservice(auth 26252..26331) post_refresh
2017-11-05 21:59:12.691490 7f2338baee00 10 mon.ceph03@-1(probing).paxosservice(mgr 1..394) post_refresh
2017-11-05 21:59:12.691491 7f2338baee00 10 mon.ceph03@-1(probing).paxosservice(mgrstat 56609513..56609621) post_refresh
2017-11-05 21:59:12.691493 7f2338baee00 10 mon.ceph03@-1(probing).paxosservice(health 1..22) post_refresh
2017-11-05 21:59:12.691528 7f2338baee00 10 mon.ceph03@-1(probing).health(0) init
2017-11-05 21:59:12.691685 7f2338baee00  2 auth: KeyRing::load: loaded key file /var/lib/ceph/mon/ceph-ceph03/keyring
2017-11-05 21:59:12.691809 7f2338baee00  2 mon.ceph03@-1(probing) e14 init
2017-11-05 21:59:12.691941 7f232d2de700  0 -- 10.10.1.23:0/15195 >> 10.10.1.21:6806/22556 conn(0x7f2342dab800 :-1 s=STATE_CONNECTING_WAIT_CONNECT_REPLY_AUTH pgs=0 cs=0 l=0).handle_connect_reply connect got BADAUTHORIZER
2017-11-05 21:59:12.692032 7f23322e8700 20 _process 0x7f2342d4ae00 1 [0,512)
2017-11-05 21:59:12.692034 7f2331ae7700 20 _process 0x7f2342d4ae00 2 [0,128)
2017-11-05 21:59:12.692076 7f23312e6700 20 _process 0x7f2342d4ae00 4 [0,128)
2017-11-05 21:59:12.692193 7f2338baee00 10 mon.ceph03@-1(probing) e14 bootstrap
2017-11-05 21:59:12.692201 7f2338baee00 10 mon.ceph03@-1(probing) e14 sync_reset_requester
2017-11-05 21:59:12.692203 7f2338baee00 10 mon.ceph03@-1(probing) e14 unregister_cluster_logger - not registered
2017-11-05 21:59:12.692204 7f2338baee00 10 mon.ceph03@-1(probing) e14 cancel_probe_timeout (none scheduled)
2017-11-05 21:59:12.692208 7f2338baee00  0 mon.ceph03@-1(probing) e14  my rank is now 2 (was -1)
2017-11-05 21:59:12.692217 7f2338baee00 10 mon.ceph03@2(probing) e14 _reset
2017-11-05 21:59:12.692219 7f2338baee00 10 mon.ceph03@2(probing) e14 cancel_probe_timeout (none scheduled)
2017-11-05 21:59:12.692220 7f2338baee00 10 mon.ceph03@2(probing) e14 timecheck_finish
2017-11-05 21:59:12.692223 7f2338baee00 15 mon.ceph03@2(probing) e14 health_tick_stop
2017-11-05 21:59:12.692224 7f2338baee00 15 mon.ceph03@2(probing) e14 health_interval_stop
2017-11-05 21:59:12.692226 7f2338baee00 10 mon.ceph03@2(probing) e14 scrub_event_cancel
2017-11-05 21:59:12.692230 7f2338baee00 10 mon.ceph03@2(probing) e14 scrub_reset
2017-11-05 21:59:12.692232 7f2338baee00 10 mon.ceph03@2(probing).paxos(paxos recovering c 112907080..112907717) restart -- canceling timeouts
2017-11-05 21:59:12.692238 7f2338baee00 10 mon.ceph03@2(probing).paxosservice(pgmap 56600067..56600783) restart
2017-11-05 21:59:12.692241 7f2338baee00 10 mon.ceph03@2(probing).paxosservice(mdsmap 1..1) restart
2017-11-05 21:59:12.692243 7f2338baee00 10 mon.ceph03@2(probing).paxosservice(osdmap 503..1105) restart
2017-11-05 21:59:12.692245 7f2338baee00 10 mon.ceph03@2(probing).paxosservice(logm 55596915..55597627) restart
2017-11-05 21:59:12.692247 7f2338baee00 10 mon.ceph03@2(probing).paxosservice(monmap 1..14) restart
2017-11-05 21:59:12.692248 7f2338baee00 10 mon.ceph03@2(probing).paxosservice(auth 26252..26331) restart
2017-11-05 21:59:12.692250 7f2338baee00 10 mon.ceph03@2(probing).paxosservice(mgr 1..394) restart
2017-11-05 21:59:12.692252 7f2338baee00 10 mon.ceph03@2(probing).paxosservice(mgrstat 56609513..56609621) restart
2017-11-05 21:59:12.692253 7f2338baee00 10 mon.ceph03@2(probing).paxosservice(health 1..22) restart
2017-11-05 21:59:12.692257 7f2338baee00 10 mon.ceph03@2(probing) e14 cancel_probe_timeout (none scheduled)
2017-11-05 21:59:12.692260 7f2338baee00 10 mon.ceph03@2(probing) e14 reset_probe_timeout 0x7f2342ba0340 after 2 seconds
2017-11-05 21:59:12.692279 7f2338baee00 10 mon.ceph03@2(probing) e14 probing other monitors
2017-11-05 21:59:12.692522 7f232d2de700  0 -- 10.10.1.23:0/15195 >> 10.10.1.21:6806/22556 conn(0x7f2342dab800 :-1 s=STATE_CONNECTING_WAIT_CONNECT_REPLY_AUTH pgs=0 cs=0 l=0).handle_connect_reply connect got BADAUTHORIZER
2017-11-05 21:59:12.692820 7f232c2dc700 10 mon.ceph03@2(probing) e14 ms_get_authorizer for mon
2017-11-05 21:59:12.692931 7f232c2dc700 10 cephx: build_service_ticket service mon secret_id 18446744073709551615 ticket_info.ticket.name=mon.
2017-11-05 21:59:12.692929 7f232cadd700 10 mon.ceph03@2(probing) e14 ms_get_authorizer for mon
2017-11-05 21:59:12.693026 7f232cadd700 10 cephx: build_service_ticket service mon secret_id 18446744073709551615 ticket_info.ticket.name=mon.
2017-11-05 21:59:12.693394 7f232cadd700 10 mon.ceph03@2(probing) e14 ms_get_authorizer for mon
2017-11-05 21:59:12.693418 7f232c2dc700 10 mon.ceph03@2(probing) e14 ms_get_authorizer for mon
2017-11-05 21:59:12.693504 7f232c2dc700 10 cephx: build_service_ticket service mon secret_id 18446744073709551615 ticket_info.ticket.name=mon.
2017-11-05 21:59:12.693751 7f232cadd700 10 cephx: build_service_ticket service mon secret_id 18446744073709551615 ticket_info.ticket.name=mon.
2017-11-05 21:59:12.694011 7f23322e8700 10 osdmap epoch 1105 mapping took 0.004593 seconds
2017-11-05 21:59:12.694022 7f23322e8700 10 mon.ceph03@2(probing).osd e1105 update_creating_pgs 0 pgs creating, 0 pools in queue
2017-11-05 21:59:12.694142 7f232c2dc700 10 mon.ceph03@2(probing) e14 ms_get_authorizer for mon
2017-11-05 21:59:12.694190 7f232cadd700 10 mon.ceph03@2(probing) e14 ms_get_authorizer for mon
2017-11-05 21:59:12.694201 7f232c2dc700 10 cephx: build_service_ticket service mon secret_id 18446744073709551615 ticket_info.ticket.name=mon.
2017-11-05 21:59:12.694246 7f232cadd700 10 cephx: build_service_ticket service mon secret_id 18446744073709551615 ticket_info.ticket.name=mon.
2017-11-05 21:59:12.694485 7f232c2dc700 10 In get_auth_session_handler for protocol 2
2017-11-05 21:59:12.694490 7f232cadd700 10 In get_auth_session_handler for protocol 2
2017-11-05 21:59:12.694558 7f232c2dc700 10 _calc_signature seq 1 front_crc_ = 637078008 middle_crc = 0 data_crc = 0 sig = 8409844418224713746
2017-11-05 21:59:12.694569 7f232c2dc700 20 Putting signature in client message(seq # 1): sig = 8409844418224713746
2017-11-05 21:59:12.694578 7f232cadd700 10 _calc_signature seq 1 front_crc_ = 637078008 middle_crc = 0 data_crc = 0 sig = 16785736216537150546
2017-11-05 21:59:12.694584 7f232cadd700 20 Putting signature in client message(seq # 1): sig = 16785736216537150546
2017-11-05 21:59:12.694882 7f232c2dc700 10 _calc_signature seq 248515223 front_crc_ = 750666649 middle_crc = 0 data_crc = 0 sig = 5792814148247093935
2017-11-05 21:59:12.694892 7f232c2dc700  0 -- 10.10.1.23:6789/0 >> 10.10.1.21:6789/0 conn(0x7f2342dd4800 :-1 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=1124301 cs=1 l=0).process missed message?  skipped from seq 0 to 248515223
2017-11-05 21:59:12.694997 7f23302e4700 10 mon.ceph03@2(probing) e14 _ms_dispatch new session 0x7f2342930900 MonSession(mon.0 10.10.1.21:6789/0 is open ) features 0x1ffddff8eea4fffb
2017-11-05 21:59:12.695008 7f23302e4700  5 mon.ceph03@2(probing) e14 _ms_dispatch setting monitor caps on this connection
2017-11-05 21:59:12.695032 7f23302e4700 20 mon.ceph03@2(probing) e14  caps allow *
2017-11-05 21:59:12.695040 7f23302e4700 20 is_capable service=mon command= read on cap allow *
2017-11-05 21:59:12.695043 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:12.695045 7f23302e4700 20  allow all
2017-11-05 21:59:12.695047 7f23302e4700 10 mon.ceph03@2(probing) e14 handle_probe mon_probe(reply 726a42b3-ae4c-4d0b-82c5-0a9065711b5f name ceph01mon quorum 0,1 paxos( fc 112907080 lc 112907738 )) v6
2017-11-05 21:59:12.695052 7f23302e4700 10 mon.ceph03@2(probing) e14 handle_probe_reply mon.0 10.10.1.21:6789/0mon_probe(reply 726a42b3-ae4c-4d0b-82c5-0a9065711b5f name ceph01mon quorum 0,1 paxos( fc 112907080 lc 112907738 )) v6
2017-11-05 21:59:12.695059 7f23302e4700 10 mon.ceph03@2(probing) e14  monmap is e14: 3 mons at {ceph01mon=10.10.1.21:6789/0,ceph02mon=10.10.1.22:6789/0,ceph03=10.10.1.23:6789/0}
2017-11-05 21:59:12.695100 7f23302e4700 10 mon.ceph03@2(probing) e14  peer name is ceph01mon
2017-11-05 21:59:12.695107 7f23302e4700 10 mon.ceph03@2(probing) e14  peer paxos last version 112907738 vs my version 112907717 (too far ahead)
2017-11-05 21:59:12.695111 7f23302e4700 10 mon.ceph03@2(probing) e14 cancel_probe_timeout 0x7f2342ba0340
2017-11-05 21:59:12.695115 7f23302e4700 10 mon.ceph03@2(probing) e14 sync_start mon.0 10.10.1.21:6789/0 recent
2017-11-05 21:59:12.695122 7f23302e4700 10 mon.ceph03@2(synchronizing) e14 sync_reset_provider
2017-11-05 21:59:12.695125 7f23302e4700 10 mon.ceph03@2(synchronizing) e14 sync_reset_timeout
2017-11-05 21:59:12.695183 7f232c2dc700 10 _calc_signature seq 2 front_crc_ = 2718256703 middle_crc = 0 data_crc = 0 sig = 4541292504390853050
2017-11-05 21:59:12.695193 7f232c2dc700 20 Putting signature in client message(seq # 2): sig = 4541292504390853050
2017-11-05 21:59:12.695462 7f232c2dc700 10 _calc_signature seq 248515224 front_crc_ = 2747452703 middle_crc = 0 data_crc = 0 sig = 10051698588873077381
2017-11-05 21:59:12.695521 7f23302e4700 20 mon.ceph03@2(synchronizing) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
2017-11-05 21:59:12.695529 7f23302e4700 20 mon.ceph03@2(synchronizing) e14  caps allow *
2017-11-05 21:59:12.695532 7f23302e4700 20 is_capable service=mon command= read on cap allow *
2017-11-05 21:59:12.695534 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:12.695535 7f23302e4700 20  allow all
2017-11-05 21:59:12.695537 7f23302e4700 10 mon.ceph03@2(synchronizing) e14 handle_sync mon_sync(cookie cookie 5737807875 lc 112907717) v2
2017-11-05 21:59:12.695540 7f23302e4700 10 mon.ceph03@2(synchronizing) e14 handle_sync_cookie mon_sync(cookie cookie 5737807875 lc 112907717) v2
2017-11-05 21:59:12.695555 7f23302e4700 10 mon.ceph03@2(synchronizing) e14 sync_reset_timeout
2017-11-05 21:59:12.695559 7f23302e4700 20 mon.ceph03@2(synchronizing) e14 sync_get_next_chunk cookie 5737807875 provider mon.0 10.10.1.21:6789/0
2017-11-05 21:59:12.695612 7f232c2dc700 10 _calc_signature seq 3 front_crc_ = 2405319517 middle_crc = 0 data_crc = 0 sig = 7150424104754382822
2017-11-05 21:59:12.695621 7f232c2dc700 20 Putting signature in client message(seq # 3): sig = 7150424104754382822
2017-11-05 21:59:12.697237 7f232c2dc700 10 _calc_signature seq 248515225 front_crc_ = 711971838 middle_crc = 0 data_crc = 0 sig = 17556540721934124985
2017-11-05 21:59:12.697338 7f23302e4700 20 mon.ceph03@2(synchronizing) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
2017-11-05 21:59:12.697348 7f23302e4700 20 mon.ceph03@2(synchronizing) e14  caps allow *
2017-11-05 21:59:12.697351 7f23302e4700 20 is_capable service=mon command= read on cap allow *
2017-11-05 21:59:12.697354 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:12.697355 7f23302e4700 20  allow all
2017-11-05 21:59:12.697357 7f23302e4700 10 mon.ceph03@2(synchronizing) e14 handle_sync mon_sync(last_chunk cookie 5737807875 lc 112907738 bl 112078 bytes) v2
2017-11-05 21:59:12.697362 7f23302e4700 10 mon.ceph03@2(synchronizing) e14 handle_sync_chunk mon_sync(last_chunk cookie 5737807875 lc 112907738 bl 112078 bytes) v2
2017-11-05 21:59:12.701065 7f232cadd700 10 _calc_signature seq 123007630 front_crc_ = 2407865230 middle_crc = 0 data_crc = 0 sig = 13627423979967733608
2017-11-05 21:59:12.701085 7f232cadd700  0 -- 10.10.1.23:6789/0 >> 10.10.1.22:6789/0 conn(0x7f2342dd3000 :-1 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=973717 cs=1 l=0).process missed message?  skipped from seq 0 to 123007630
2017-11-05 21:59:12.751656 7f23302e4700 10 mon.ceph03@2(synchronizing) e14 handle_sync_chunk applying recent paxos transactions as we go
2017-11-05 21:59:12.751676 7f23302e4700 10 mon.ceph03@2(synchronizing).paxos(paxos recovering c 112907080..112907717) read_and_prepare_transactions first 112907718 last 112907738
2017-11-05 21:59:12.751915 7f23302e4700 15 mon.ceph03@2(synchronizing).paxos(paxos recovering c 112907080..112907717) read_and_prepare_transactions total versions 20
2017-11-05 21:59:12.796201 7f23302e4700 10 mon.ceph03@2(synchronizing).paxos(paxos recovering c 112907080..112907738) init last_pn: 12300 accepted_pn: 13500 last_committed: 112907738 first_committed: 112907080
2017-11-05 21:59:12.796218 7f23302e4700 10 mon.ceph03@2(synchronizing).paxos(paxos recovering c 112907080..112907738) init
2017-11-05 21:59:12.796243 7f23302e4700 10 mon.ceph03@2(synchronizing) e14 sync_finish lc 112907738 from mon.0 10.10.1.21:6789/0
2017-11-05 21:59:12.840316 7f23302e4700 10 mon.ceph03@2(synchronizing) e14 init_paxos
2017-11-05 21:59:12.840411 7f23302e4700 10 mon.ceph03@2(synchronizing).paxos(paxos recovering c 112907080..112907738) init last_pn: 12300 accepted_pn: 13500 last_committed: 112907738 first_committed: 112907080
2017-11-05 21:59:12.840421 7f23302e4700 10 mon.ceph03@2(synchronizing).paxos(paxos recovering c 112907080..112907738) init
2017-11-05 21:59:12.840439 7f23302e4700  1 mon.ceph03@2(synchronizing).mds e1 Unable to load 'last_metadata'
2017-11-05 21:59:12.840477 7f23302e4700 10 mon.ceph03@2(synchronizing).health init
2017-11-05 21:59:12.840482 7f23302e4700 10 mon.ceph03@2(synchronizing) e14 refresh_from_paxos
2017-11-05 21:59:12.840496 7f23302e4700 10 mon.ceph03@2(synchronizing) e14 refresh_from_paxos no cluster_fingerprint
2017-11-05 21:59:12.840526 7f23302e4700 10 mon.ceph03@2(synchronizing).paxosservice(pgmap 56600067..56600783) refresh
2017-11-05 21:59:12.840563 7f23302e4700 10 mon.ceph03@2(synchronizing).paxosservice(mdsmap 1..1) refresh
2017-11-05 21:59:12.840602 7f23302e4700 10 mon.ceph03@2(synchronizing).paxosservice(osdmap 503..1105) refresh
2017-11-05 21:59:12.840629 7f23302e4700 10 mon.ceph03@2(synchronizing).paxosservice(logm 55596915..55597634) refresh
2017-11-05 21:59:12.840634 7f23302e4700 10 mon.ceph03@2(synchronizing).log v55597634 update_from_paxos
2017-11-05 21:59:12.840644 7f23302e4700 10 mon.ceph03@2(synchronizing).log v55597634 update_from_paxos version 55597634 summary v 55597627
2017-11-05 21:59:12.840655 7f23302e4700 10 mon.ceph03@2(synchronizing).log v55597634 update_from_paxos latest full 55597633
2017-11-05 21:59:12.840669 7f23302e4700  7 mon.ceph03@2(synchronizing).log v55597634 update_from_paxos loading summary e55597633
2017-11-05 21:59:12.840809 7f23302e4700  7 mon.ceph03@2(synchronizing).log v55597634 update_from_paxos loaded summary e55597633
2017-11-05 21:59:12.840824 7f23302e4700  7 mon.ceph03@2(synchronizing).log v55597634 update_from_paxos applying incremental log 55597634 2017-11-05 21:59:10.716523 mgr.ceph01mgr client.54805773 10.10.1.21:0/1619113657 1817 : audit [DBG] from='client.55026393 10.10.1.21:0/2834401706' entity='client.admin' cmd=[{"prefix": "osd pool stats", "target": ["mgr", ""], "format": "json"}]: dispatch
2017-11-05 21:59:12.840844 7f23302e4700 20 mon.ceph03@2(synchronizing).log v55597634 update_from_paxos logging for channel 'audit' to file '/var/log/ceph/ceph.audit.log'
2017-11-05 21:59:12.840853 7f23302e4700 15 mon.ceph03@2(synchronizing).log v55597634 update_from_paxos logging for 0 channels
2017-11-05 21:59:12.840858 7f23302e4700 10 mon.ceph03@2(synchronizing).log v55597634 check_subs
2017-11-05 21:59:12.840895 7f23302e4700 10 mon.ceph03@2(synchronizing).paxosservice(monmap 1..14) refresh
2017-11-05 21:59:12.840921 7f23302e4700 10 mon.ceph03@2(synchronizing).paxosservice(auth 26252..26331) refresh
2017-11-05 21:59:12.840926 7f23302e4700 10 mon.ceph03@2(synchronizing).auth v26331 update_from_paxos
2017-11-05 21:59:12.840960 7f23302e4700 10 mon.ceph03@2(synchronizing).paxosservice(mgr 1..394) refresh
2017-11-05 21:59:12.841022 7f23302e4700 10 mon.ceph03@2(synchronizing).paxosservice(mgrstat 56609513..56609633) refresh
2017-11-05 21:59:12.841027 7f23302e4700 10 mon.ceph03@2(synchronizing).mgrstat  56609633
2017-11-05 21:59:12.841063 7f23302e4700 10 mon.ceph03@2(synchronizing).mgrstat update_from_paxos v56609633 service_map e1
2017-11-05 21:59:12.841068 7f23302e4700 10 mon.ceph03@2(synchronizing).mgrstat check_subs
2017-11-05 21:59:12.841070 7f23302e4700 20 mon.ceph03@2(synchronizing).mgrstat update_logger
2017-11-05 21:59:12.841097 7f23302e4700 10 mon.ceph03@2(synchronizing).paxosservice(health 1..23) refresh
2017-11-05 21:59:12.841104 7f23302e4700 10 mon.ceph03@2(synchronizing).health update_from_paxos
2017-11-05 21:59:12.841121 7f23302e4700 20 mon.ceph03@2(synchronizing).health dump:{
    "quorum_health": {},
    "leader_health": {
        "MON_DOWN": {
            "severity": "HEALTH_WARN",
            "summary": {
                "message": "1/3 mons down, quorum ceph01mon,ceph02mon"
            },
            "detail": [
                {
                    "message": "mon.ceph03 (rank 2) addr 10.10.1.23:6789/0 is down (out of quorum)"
                }
            ]
        }
    }
}

2017-11-05 21:59:12.841151 7f23302e4700 10 mon.ceph03@2(synchronizing).paxosservice(pgmap 56600067..56600783) post_refresh
2017-11-05 21:59:12.841154 7f23302e4700 10 mon.ceph03@2(synchronizing).paxosservice(mdsmap 1..1) post_refresh
2017-11-05 21:59:12.841155 7f23302e4700 10 mon.ceph03@2(synchronizing).paxosservice(osdmap 503..1105) post_refresh
2017-11-05 21:59:12.841156 7f23302e4700 10 mon.ceph03@2(synchronizing).paxosservice(logm 55596915..55597634) post_refresh
2017-11-05 21:59:12.841158 7f23302e4700 10 mon.ceph03@2(synchronizing).paxosservice(monmap 1..14) post_refresh
2017-11-05 21:59:12.841159 7f23302e4700 10 mon.ceph03@2(synchronizing).paxosservice(auth 26252..26331) post_refresh
2017-11-05 21:59:12.841162 7f23302e4700 10 mon.ceph03@2(synchronizing).paxosservice(mgr 1..394) post_refresh
2017-11-05 21:59:12.841163 7f23302e4700 10 mon.ceph03@2(synchronizing).paxosservice(mgrstat 56609513..56609633) post_refresh
2017-11-05 21:59:12.841165 7f23302e4700 10 mon.ceph03@2(synchronizing).paxosservice(health 1..23) post_refresh
2017-11-05 21:59:12.841212 7f23302e4700 10 mon.ceph03@2(synchronizing) e14 bootstrap
2017-11-05 21:59:12.841215 7f23302e4700 10 mon.ceph03@2(synchronizing) e14 sync_reset_requester
2017-11-05 21:59:12.841218 7f23302e4700 10 mon.ceph03@2(synchronizing) e14 unregister_cluster_logger - not registered
2017-11-05 21:59:12.841219 7f23302e4700 10 mon.ceph03@2(synchronizing) e14 cancel_probe_timeout (none scheduled)
2017-11-05 21:59:12.841222 7f23302e4700 10 mon.ceph03@2(probing) e14 _reset
2017-11-05 21:59:12.841223 7f23302e4700 10 mon.ceph03@2(probing) e14 cancel_probe_timeout (none scheduled)
2017-11-05 21:59:12.841224 7f23302e4700 10 mon.ceph03@2(probing) e14 timecheck_finish
2017-11-05 21:59:12.841226 7f23302e4700 15 mon.ceph03@2(probing) e14 health_tick_stop
2017-11-05 21:59:12.841227 7f23302e4700 15 mon.ceph03@2(probing) e14 health_interval_stop
2017-11-05 21:59:12.841228 7f23302e4700 10 mon.ceph03@2(probing) e14 scrub_event_cancel
2017-11-05 21:59:12.841230 7f23302e4700 10 mon.ceph03@2(probing) e14 scrub_reset
2017-11-05 21:59:12.841232 7f23302e4700 10 mon.ceph03@2(probing).paxos(paxos recovering c 112907080..112907738) restart -- canceling timeouts
2017-11-05 21:59:12.841235 7f23302e4700 10 mon.ceph03@2(probing).paxosservice(pgmap 56600067..56600783) restart
2017-11-05 21:59:12.841237 7f23302e4700 10 mon.ceph03@2(probing).paxosservice(mdsmap 1..1) restart
2017-11-05 21:59:12.841239 7f23302e4700 10 mon.ceph03@2(probing).paxosservice(osdmap 503..1105) restart
2017-11-05 21:59:12.841242 7f23302e4700 10 mon.ceph03@2(probing).paxosservice(logm 55596915..55597634) restart
2017-11-05 21:59:12.841243 7f23302e4700 10 mon.ceph03@2(probing).paxosservice(monmap 1..14) restart
2017-11-05 21:59:12.841244 7f23302e4700 10 mon.ceph03@2(probing).paxosservice(auth 26252..26331) restart
2017-11-05 21:59:12.841246 7f23302e4700 10 mon.ceph03@2(probing).paxosservice(mgr 1..394) restart
2017-11-05 21:59:12.841247 7f23302e4700 10 mon.ceph03@2(probing).paxosservice(mgrstat 56609513..56609633) restart
2017-11-05 21:59:12.841250 7f23302e4700 10 mon.ceph03@2(probing).paxosservice(health 1..23) restart
2017-11-05 21:59:12.841253 7f23302e4700 10 mon.ceph03@2(probing) e14 cancel_probe_timeout (none scheduled)
2017-11-05 21:59:12.841260 7f23302e4700 10 mon.ceph03@2(probing) e14 reset_probe_timeout 0x7f2342ba02b0 after 2 seconds
2017-11-05 21:59:12.841269 7f23302e4700 10 mon.ceph03@2(probing) e14 probing other monitors
2017-11-05 21:59:12.841369 7f23302e4700 10 mon.ceph03@2(probing) e14 _ms_dispatch new session 0x7f2342930c00 MonSession(mon.1 10.10.1.22:6789/0 is open ) features 0x1ffddff8eea4fffb
2017-11-05 21:59:12.841378 7f23302e4700  5 mon.ceph03@2(probing) e14 _ms_dispatch setting monitor caps on this connection
2017-11-05 21:59:12.841381 7f23302e4700 20 mon.ceph03@2(probing) e14  caps allow *
2017-11-05 21:59:12.841376 7f232c2dc700 10 _calc_signature seq 4 front_crc_ = 637078008 middle_crc = 0 data_crc = 0 sig = 486809900323904982
2017-11-05 21:59:12.841385 7f23302e4700 20 is_capable service=mon command= read on cap allow *
2017-11-05 21:59:12.841381 7f232cadd700 10 _calc_signature seq 2 front_crc_ = 637078008 middle_crc = 0 data_crc = 0 sig = 113191317311269439
2017-11-05 21:59:12.841387 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:12.841388 7f23302e4700 20  allow all
2017-11-05 21:59:12.841388 7f232c2dc700 20 Putting signature in client message(seq # 4): sig = 486809900323904982
2017-11-05 21:59:12.841389 7f232cadd700 20 Putting signature in client message(seq # 2): sig = 113191317311269439
2017-11-05 21:59:12.841393 7f23302e4700 10 mon.ceph03@2(probing) e14 handle_probe mon_probe(reply 726a42b3-ae4c-4d0b-82c5-0a9065711b5f name ceph02mon quorum 0,1 paxos( fc 112907080 lc 112907738 )) v6
2017-11-05 21:59:12.841397 7f23302e4700 10 mon.ceph03@2(probing) e14 handle_probe_reply mon.1 10.10.1.22:6789/0mon_probe(reply 726a42b3-ae4c-4d0b-82c5-0a9065711b5f name ceph02mon quorum 0,1 paxos( fc 112907080 lc 112907738 )) v6
2017-11-05 21:59:12.841401 7f23302e4700 10 mon.ceph03@2(probing) e14  monmap is e14: 3 mons at {ceph01mon=10.10.1.21:6789/0,ceph02mon=10.10.1.22:6789/0,ceph03=10.10.1.23:6789/0}
2017-11-05 21:59:12.841420 7f23302e4700 10 mon.ceph03@2(probing) e14  peer name is ceph02mon
2017-11-05 21:59:12.841422 7f23302e4700 10 mon.ceph03@2(probing) e14  existing quorum 0,1
2017-11-05 21:59:12.841424 7f23302e4700 10 mon.ceph03@2(probing) e14  peer paxos version 112907738 vs my version 112907738 (ok)
2017-11-05 21:59:12.841427 7f23302e4700 10 mon.ceph03@2(probing) e14 start_election
2017-11-05 21:59:12.841428 7f23302e4700 10 mon.ceph03@2(probing) e14 _reset
2017-11-05 21:59:12.841429 7f23302e4700 10 mon.ceph03@2(probing) e14 cancel_probe_timeout 0x7f2342ba02b0
2017-11-05 21:59:12.841431 7f23302e4700 10 mon.ceph03@2(probing) e14 timecheck_finish
2017-11-05 21:59:12.841433 7f23302e4700 15 mon.ceph03@2(probing) e14 health_tick_stop
2017-11-05 21:59:12.841434 7f23302e4700 15 mon.ceph03@2(probing) e14 health_interval_stop
2017-11-05 21:59:12.841435 7f23302e4700 10 mon.ceph03@2(probing) e14 scrub_event_cancel
2017-11-05 21:59:12.841436 7f23302e4700 10 mon.ceph03@2(probing) e14 scrub_reset
2017-11-05 21:59:12.841440 7f23302e4700 10 mon.ceph03@2(probing).paxos(paxos recovering c 112907080..112907738) restart -- canceling timeouts
2017-11-05 21:59:12.841443 7f23302e4700 10 mon.ceph03@2(probing).paxosservice(pgmap 56600067..56600783) restart
2017-11-05 21:59:12.841445 7f23302e4700 10 mon.ceph03@2(probing).paxosservice(mdsmap 1..1) restart
2017-11-05 21:59:12.841446 7f23302e4700 10 mon.ceph03@2(probing).paxosservice(osdmap 503..1105) restart
2017-11-05 21:59:12.841448 7f23302e4700 10 mon.ceph03@2(probing).paxosservice(logm 55596915..55597634) restart
2017-11-05 21:59:12.841449 7f23302e4700 10 mon.ceph03@2(probing).paxosservice(monmap 1..14) restart
2017-11-05 21:59:12.841450 7f23302e4700 10 mon.ceph03@2(probing).paxosservice(auth 26252..26331) restart
2017-11-05 21:59:12.841452 7f23302e4700 10 mon.ceph03@2(probing).paxosservice(mgr 1..394) restart
2017-11-05 21:59:12.841453 7f23302e4700 10 mon.ceph03@2(probing).paxosservice(mgrstat 56609513..56609633) restart
2017-11-05 21:59:12.841454 7f23302e4700 10 mon.ceph03@2(probing).paxosservice(health 1..23) restart
2017-11-05 21:59:12.841466 7f23302e4700  0 log_channel(cluster) log [INF] : mon.ceph03 calling new monitor election
2017-11-05 21:59:12.841494 7f23302e4700  5 mon.ceph03@2(electing).elector(0) start -- can i be leader?
2017-11-05 21:59:12.841515 7f23302e4700  1 mon.ceph03@2(electing).elector(338) init, last seen epoch 338
2017-11-05 21:59:12.841517 7f23302e4700 10 mon.ceph03@2(electing).elector(338) bump_epoch 338 to 339
2017-11-05 21:59:12.841793 7f232c2dc700 10 _calc_signature seq 248515226 front_crc_ = 750666649 middle_crc = 0 data_crc = 0 sig = 8049785851673397625
2017-11-05 21:59:12.841878 7f232cadd700 10 _calc_signature seq 123007631 front_crc_ = 2407865230 middle_crc = 0 data_crc = 0 sig = 7765281711407537163
2017-11-05 21:59:12.884793 7f23302e4700 10 mon.ceph03@2(electing) e14 join_election
2017-11-05 21:59:12.884808 7f23302e4700 10 mon.ceph03@2(electing) e14 _reset
2017-11-05 21:59:12.884811 7f23302e4700 10 mon.ceph03@2(electing) e14 cancel_probe_timeout (none scheduled)
2017-11-05 21:59:12.884813 7f23302e4700 10 mon.ceph03@2(electing) e14 timecheck_finish
2017-11-05 21:59:12.884816 7f23302e4700 15 mon.ceph03@2(electing) e14 health_tick_stop
2017-11-05 21:59:12.884817 7f23302e4700 15 mon.ceph03@2(electing) e14 health_interval_stop
2017-11-05 21:59:12.884819 7f23302e4700 10 mon.ceph03@2(electing) e14 scrub_event_cancel
2017-11-05 21:59:12.884821 7f23302e4700 10 mon.ceph03@2(electing) e14 scrub_reset
2017-11-05 21:59:12.884823 7f23302e4700 10 mon.ceph03@2(electing).paxos(paxos recovering c 112907080..112907738) restart -- canceling timeouts
2017-11-05 21:59:12.884828 7f23302e4700 10 mon.ceph03@2(electing).paxosservice(pgmap 56600067..56600783) restart
2017-11-05 21:59:12.884830 7f23302e4700 10 mon.ceph03@2(electing).paxosservice(mdsmap 1..1) restart
2017-11-05 21:59:12.884832 7f23302e4700 10 mon.ceph03@2(electing).paxosservice(osdmap 503..1105) restart
2017-11-05 21:59:12.884834 7f23302e4700 10 mon.ceph03@2(electing).paxosservice(logm 55596915..55597634) restart
2017-11-05 21:59:12.884836 7f23302e4700 10 mon.ceph03@2(electing).paxosservice(monmap 1..14) restart
2017-11-05 21:59:12.884840 7f23302e4700 10 mon.ceph03@2(electing).paxosservice(auth 26252..26331) restart
2017-11-05 21:59:12.884842 7f23302e4700 10 mon.ceph03@2(electing).paxosservice(mgr 1..394) restart
2017-11-05 21:59:12.884844 7f23302e4700 10 mon.ceph03@2(electing).paxosservice(mgrstat 56609513..56609633) restart
2017-11-05 21:59:12.884845 7f23302e4700 10 mon.ceph03@2(electing).paxosservice(health 1..23) restart
2017-11-05 21:59:12.885354 7f232c2dc700 10 _calc_signature seq 5 front_crc_ = 2237315324 middle_crc = 0 data_crc = 0 sig = 12691232392794583066
2017-11-05 21:59:12.885380 7f232c2dc700 20 Putting signature in client message(seq # 5): sig = 12691232392794583066
2017-11-05 21:59:12.885401 7f23302e4700 10 mon.ceph03@2(electing) e14 _ms_dispatch new session 0x7f2342930a80 MonSession(mon.2 10.10.1.23:6789/0 is open ) features 0x1ffddff8eea4fffb
2017-11-05 21:59:12.885412 7f23302e4700  5 mon.ceph03@2(electing) e14 _ms_dispatch setting monitor caps on this connection
2017-11-05 21:59:12.885407 7f232cadd700 10 _calc_signature seq 3 front_crc_ = 2237315324 middle_crc = 0 data_crc = 0 sig = 53150067798815490
2017-11-05 21:59:12.885415 7f23302e4700 20 mon.ceph03@2(electing) e14  caps allow *
2017-11-05 21:59:12.885416 7f232cadd700 20 Putting signature in client message(seq # 3): sig = 53150067798815490
2017-11-05 21:59:12.885418 7f23302e4700 10 mon.ceph03@2(electing).paxosservice(logm 55596915..55597634) dispatch 0x7f234296e300 log(1 entries from seq 1 at 2017-11-05 21:59:12.841468) v1 from mon.2 10.10.1.23:6789/0 con 0x7f23431d6000
2017-11-05 21:59:12.885427 7f23302e4700  5 mon.ceph03@2(electing).paxos(paxos recovering c 112907080..112907738) is_readable = 0 - now=2017-11-05 21:59:12.885427 lease_expire=0.000000 has v0 lc 112907738
2017-11-05 21:59:12.885433 7f23302e4700 10 mon.ceph03@2(electing).paxosservice(logm 55596915..55597634)  waiting for paxos -> readable (v0)
2017-11-05 21:59:12.885438 7f23302e4700  5 mon.ceph03@2(electing).paxos(paxos recovering c 112907080..112907738) is_readable = 0 - now=2017-11-05 21:59:12.885438 lease_expire=0.000000 has v0 lc 112907738
2017-11-05 21:59:12.885455 7f23302e4700 20 mon.ceph03@2(electing) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
2017-11-05 21:59:12.885458 7f23302e4700 20 mon.ceph03@2(electing) e14  caps allow *
2017-11-05 21:59:12.885461 7f23302e4700 20 is_capable service=mon command= read on cap allow *
2017-11-05 21:59:12.885463 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:12.885464 7f23302e4700 20  allow all
2017-11-05 21:59:12.885465 7f23302e4700 10 mon.ceph03@2(electing) e14 handle_probe mon_probe(reply 726a42b3-ae4c-4d0b-82c5-0a9065711b5f name ceph01mon quorum 0,1 paxos( fc 112907080 lc 112907738 )) v6
2017-11-05 21:59:12.885469 7f23302e4700 10 mon.ceph03@2(electing) e14 handle_probe_reply mon.0 10.10.1.21:6789/0mon_probe(reply 726a42b3-ae4c-4d0b-82c5-0a9065711b5f name ceph01mon quorum 0,1 paxos( fc 112907080 lc 112907738 )) v6
2017-11-05 21:59:12.885473 7f23302e4700 10 mon.ceph03@2(electing) e14  monmap is e14: 3 mons at {ceph01mon=10.10.1.21:6789/0,ceph02mon=10.10.1.22:6789/0,ceph03=10.10.1.23:6789/0}
2017-11-05 21:59:12.885488 7f23302e4700 10 mon.ceph03@2(electing) e14  peer name is ceph01mon
2017-11-05 21:59:12.885506 7f23302e4700 20 mon.ceph03@2(electing) e14 _ms_dispatch existing session 0x7f2342930c00 for mon.1 10.10.1.22:6789/0
2017-11-05 21:59:12.885509 7f23302e4700 20 mon.ceph03@2(electing) e14  caps allow *
2017-11-05 21:59:12.885511 7f23302e4700 20 is_capable service=mon command= read on cap allow *
2017-11-05 21:59:12.885513 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:12.885513 7f23302e4700 20  allow all
2017-11-05 21:59:12.885514 7f23302e4700 10 mon.ceph03@2(electing) e14 handle_probe mon_probe(reply 726a42b3-ae4c-4d0b-82c5-0a9065711b5f name ceph02mon quorum 0,1 paxos( fc 112907080 lc 112907738 )) v6
2017-11-05 21:59:12.885518 7f23302e4700 10 mon.ceph03@2(electing) e14 handle_probe_reply mon.1 10.10.1.22:6789/0mon_probe(reply 726a42b3-ae4c-4d0b-82c5-0a9065711b5f name ceph02mon quorum 0,1 paxos( fc 112907080 lc 112907738 )) v6
2017-11-05 21:59:12.885522 7f23302e4700 10 mon.ceph03@2(electing) e14  monmap is e14: 3 mons at {ceph01mon=10.10.1.21:6789/0,ceph02mon=10.10.1.22:6789/0,ceph03=10.10.1.23:6789/0}
2017-11-05 21:59:12.885532 7f23302e4700 10 mon.ceph03@2(electing) e14  peer name is ceph02mon
2017-11-05 21:59:12.896611 7f232d2de700 10 mon.ceph03@2(electing) e14 ms_get_authorizer for mgr
2017-11-05 21:59:12.896688 7f232d2de700 10 cephx keyserverdata: get_caps: name=mon.
2017-11-05 21:59:12.896698 7f232d2de700 10 cephx: build_service_ticket service mgr secret_id 7 ticket_info.ticket.name=mon.
2017-11-05 21:59:12.896989 7f232d2de700 10 In get_auth_session_handler for protocol 2
2017-11-05 21:59:12.897028 7f232d2de700 10 _calc_signature seq 1 front_crc_ = 539573588 middle_crc = 0 data_crc = 0 sig = 2604218379037085131
2017-11-05 21:59:12.897039 7f232d2de700 20 Putting signature in client message(seq # 1): sig = 2604218379037085131
2017-11-05 21:59:12.897285 7f232d2de700 10 _calc_signature seq 1 front_crc_ = 2791807819 middle_crc = 0 data_crc = 0 sig = 898952956200682019
2017-11-05 21:59:12.897766 7f232d2de700 10 _calc_signature seq 2 front_crc_ = 2639686472 middle_crc = 0 data_crc = 0 sig = 10048159117889152159
2017-11-05 21:59:12.897783 7f232d2de700 20 Putting signature in client message(seq # 2): sig = 10048159117889152159
2017-11-05 21:59:12.911682 7f232cadd700 10 _calc_signature seq 123007632 front_crc_ = 518921869 middle_crc = 0 data_crc = 0 sig = 16547273314806196539
2017-11-05 21:59:12.911796 7f23302e4700 20 mon.ceph03@2(electing) e14 _ms_dispatch existing session 0x7f2342930c00 for mon.1 10.10.1.22:6789/0
2017-11-05 21:59:12.911805 7f23302e4700 20 mon.ceph03@2(electing) e14  caps allow *
2017-11-05 21:59:12.911808 7f23302e4700 20 is_capable service=mon command= read on cap allow *
2017-11-05 21:59:12.911811 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:12.911812 7f23302e4700 20  allow all
2017-11-05 21:59:12.911814 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
2017-11-05 21:59:12.911815 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:12.911817 7f23302e4700 20  allow all
2017-11-05 21:59:12.911839 7f23302e4700  5 mon.ceph03@2(electing).elector(339) handle_propose from mon.1
2017-11-05 21:59:12.911842 7f23302e4700 10 mon.ceph03@2(electing).elector(339) handle_propose required features 153140804152475648 mon_feature_t([kraken,luminous]), peer features 2305244844532236283 mon_feature_t([kraken,luminous])
2017-11-05 21:59:12.911848 7f23302e4700 10 mon.ceph03@2(electing).elector(339) bump_epoch 339 to 343
2017-11-05 21:59:12.920536 7f232c2dc700 10 _calc_signature seq 248515227 front_crc_ = 518921869 middle_crc = 0 data_crc = 0 sig = 326737325304326689
2017-11-05 21:59:12.951319 7f23302e4700 10 mon.ceph03@2(electing) e14 join_election
2017-11-05 21:59:12.951334 7f23302e4700 10 mon.ceph03@2(electing) e14 _reset
2017-11-05 21:59:12.951336 7f23302e4700 10 mon.ceph03@2(electing) e14 cancel_probe_timeout (none scheduled)
2017-11-05 21:59:12.951337 7f23302e4700 10 mon.ceph03@2(electing) e14 timecheck_finish
2017-11-05 21:59:12.951339 7f23302e4700 15 mon.ceph03@2(electing) e14 health_tick_stop
2017-11-05 21:59:12.951340 7f23302e4700 15 mon.ceph03@2(electing) e14 health_interval_stop
2017-11-05 21:59:12.951342 7f23302e4700 10 mon.ceph03@2(electing) e14 scrub_event_cancel
2017-11-05 21:59:12.951343 7f23302e4700 10 mon.ceph03@2(electing) e14 scrub_reset
2017-11-05 21:59:12.951346 7f23302e4700 10 mon.ceph03@2(electing).paxos(paxos recovering c 112907080..112907738) restart -- canceling timeouts
2017-11-05 21:59:12.951350 7f23302e4700 10 mon.ceph03@2(electing).paxosservice(pgmap 56600067..56600783) restart
2017-11-05 21:59:12.951353 7f23302e4700 10 mon.ceph03@2(electing).paxosservice(mdsmap 1..1) restart
2017-11-05 21:59:12.951355 7f23302e4700 10 mon.ceph03@2(electing).paxosservice(osdmap 503..1105) restart
2017-11-05 21:59:12.951356 7f23302e4700 10 mon.ceph03@2(electing).paxosservice(logm 55596915..55597634) restart
2017-11-05 21:59:12.951358 7f23302e4700 10 mon.ceph03@2(electing).paxosservice(monmap 1..14) restart
2017-11-05 21:59:12.951359 7f23302e4700 10 mon.ceph03@2(electing).paxosservice(auth 26252..26331) restart
2017-11-05 21:59:12.951361 7f23302e4700 10 mon.ceph03@2(electing).paxosservice(mgr 1..394) restart
2017-11-05 21:59:12.951363 7f23302e4700 10 mon.ceph03@2(electing).paxosservice(mgrstat 56609513..56609633) restart
2017-11-05 21:59:12.951365 7f23302e4700 10 mon.ceph03@2(electing).paxosservice(health 1..23) restart
2017-11-05 21:59:12.951375 7f23302e4700  5 mon.ceph03@2(electing).elector(343) defer to 1
2017-11-05 21:59:12.951725 7f23302e4700 20 mon.ceph03@2(electing) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
2017-11-05 21:59:12.951734 7f23302e4700 20 mon.ceph03@2(electing) e14  caps allow *
2017-11-05 21:59:12.951737 7f23302e4700 20 is_capable service=mon command= read on cap allow *
2017-11-05 21:59:12.951739 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:12.951740 7f23302e4700 20  allow all
2017-11-05 21:59:12.951741 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
2017-11-05 21:59:12.951743 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:12.951743 7f23302e4700 20  allow all
2017-11-05 21:59:12.951760 7f23302e4700  5 mon.ceph03@2(electing).elector(343) handle_propose from mon.0
2017-11-05 21:59:12.951762 7f23302e4700 10 mon.ceph03@2(electing).elector(343) handle_propose required features 153140804152475648 mon_feature_t([kraken,luminous]), peer features 2305244844532236283 mon_feature_t([kraken,luminous])
2017-11-05 21:59:12.951766 7f23302e4700  5 mon.ceph03@2(electing).elector(343) defer to 0
2017-11-05 21:59:12.951772 7f232cadd700 10 _calc_signature seq 4 front_crc_ = 2543563982 middle_crc = 0 data_crc = 0 sig = 12003399029982092639
2017-11-05 21:59:12.951781 7f232cadd700 20 Putting signature in client message(seq # 4): sig = 12003399029982092639
2017-11-05 21:59:12.952081 7f232c2dc700 10 _calc_signature seq 6 front_crc_ = 2543563982 middle_crc = 0 data_crc = 0 sig = 9154992000545278758
2017-11-05 21:59:12.952098 7f232c2dc700 20 Putting signature in client message(seq # 6): sig = 9154992000545278758
2017-11-05 21:59:12.987082 7f232c2dc700 10 _calc_signature seq 248515228 front_crc_ = 2232557517 middle_crc = 0 data_crc = 0 sig = 9253898422206336076
2017-11-05 21:59:12.987206 7f23302e4700 20 mon.ceph03@2(electing) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
2017-11-05 21:59:12.987215 7f23302e4700 20 mon.ceph03@2(electing) e14  caps allow *
2017-11-05 21:59:12.987218 7f23302e4700 20 is_capable service=mon command= read on cap allow *
2017-11-05 21:59:12.987220 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:12.987221 7f23302e4700 20  allow all
2017-11-05 21:59:12.987223 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
2017-11-05 21:59:12.987224 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:12.987225 7f23302e4700 20  allow all
2017-11-05 21:59:12.987243 7f23302e4700  5 mon.ceph03@2(electing).elector(343) handle_victory from mon.0 quorum_features 2305244844532236283 mon_feature_t([kraken,luminous])
2017-11-05 21:59:12.987248 7f23302e4700 10 mon.ceph03@2(electing).elector(343) bump_epoch 343 to 344
2017-11-05 21:59:13.029008 7f23302e4700 10 mon.ceph03@2(electing) e14 join_election
2017-11-05 21:59:13.029034 7f23302e4700 10 mon.ceph03@2(electing) e14 _reset
2017-11-05 21:59:13.029036 7f23302e4700 10 mon.ceph03@2(electing) e14 cancel_probe_timeout (none scheduled)
2017-11-05 21:59:13.029038 7f23302e4700 10 mon.ceph03@2(electing) e14 timecheck_finish
2017-11-05 21:59:13.029040 7f23302e4700 15 mon.ceph03@2(electing) e14 health_tick_stop
2017-11-05 21:59:13.029042 7f23302e4700 15 mon.ceph03@2(electing) e14 health_interval_stop
2017-11-05 21:59:13.029043 7f23302e4700 10 mon.ceph03@2(electing) e14 scrub_event_cancel
2017-11-05 21:59:13.029045 7f23302e4700 10 mon.ceph03@2(electing) e14 scrub_reset
2017-11-05 21:59:13.029047 7f23302e4700 10 mon.ceph03@2(electing).paxos(paxos recovering c 112907080..112907738) restart -- canceling timeouts
2017-11-05 21:59:13.029052 7f23302e4700 10 mon.ceph03@2(electing).paxosservice(pgmap 56600067..56600783) restart
2017-11-05 21:59:13.029056 7f23302e4700 10 mon.ceph03@2(electing).paxosservice(mdsmap 1..1) restart
2017-11-05 21:59:13.029058 7f23302e4700 10 mon.ceph03@2(electing).paxosservice(osdmap 503..1105) restart
2017-11-05 21:59:13.029060 7f23302e4700 10 mon.ceph03@2(electing).paxosservice(logm 55596915..55597634) restart
2017-11-05 21:59:13.029062 7f23302e4700 10 mon.ceph03@2(electing).paxosservice(monmap 1..14) restart
2017-11-05 21:59:13.029064 7f23302e4700 10 mon.ceph03@2(electing).paxosservice(auth 26252..26331) restart
2017-11-05 21:59:13.029065 7f23302e4700 10 mon.ceph03@2(electing).paxosservice(mgr 1..394) restart
2017-11-05 21:59:13.029067 7f23302e4700 10 mon.ceph03@2(electing).paxosservice(mgrstat 56609513..56609633) restart
2017-11-05 21:59:13.029069 7f23302e4700 10 mon.ceph03@2(electing).paxosservice(health 1..23) restart
2017-11-05 21:59:13.029075 7f23302e4700 10 mon.ceph03@2(peon) e14 lose_election, epoch 344 leader is mon0 quorum is 0,1,2 features are 2305244844532236283 mon_features are mon_feature_t([kraken,luminous])
2017-11-05 21:59:13.029083 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos recovering c 112907080..112907738) peon_init -- i am a peon
2017-11-05 21:59:13.029086 7f23302e4700 20 mon.ceph03@2(peon).paxos(paxos recovering c 112907080..112907738) reset_lease_timeout - setting timeout event
2017-11-05 21:59:13.029092 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(pgmap 56600067..56600783) election_finished
2017-11-05 21:59:13.029094 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(pgmap 56600067..56600783) _active - not active
2017-11-05 21:59:13.029097 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mdsmap 1..1) election_finished
2017-11-05 21:59:13.029099 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mdsmap 1..1) _active - not active
2017-11-05 21:59:13.029101 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(osdmap 503..1105) election_finished
2017-11-05 21:59:13.029102 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(osdmap 503..1105) _active - not active
2017-11-05 21:59:13.029104 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(logm 55596915..55597634) election_finished
2017-11-05 21:59:13.029106 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(logm 55596915..55597634) _active - not active
2017-11-05 21:59:13.029109 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(monmap 1..14) election_finished
2017-11-05 21:59:13.029111 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(monmap 1..14) _active - not active
2017-11-05 21:59:13.029113 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(auth 26252..26331) election_finished
2017-11-05 21:59:13.029114 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(auth 26252..26331) _active - not active
2017-11-05 21:59:13.029116 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgr 1..394) election_finished
2017-11-05 21:59:13.029118 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgr 1..394) _active - not active
2017-11-05 21:59:13.029119 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgrstat 56609513..56609633) election_finished
2017-11-05 21:59:13.029121 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgrstat 56609513..56609633) _active - not active
2017-11-05 21:59:13.029123 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(health 1..23) election_finished
2017-11-05 21:59:13.029124 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(health 1..23) _active - not active
2017-11-05 21:59:13.029126 7f23302e4700 10 mon.ceph03@2(peon).data_health(344) start_epoch epoch 344
2017-11-05 21:59:13.029133 7f23302e4700  5 mon.ceph03@2(peon) e14 apply_quorum_to_compatset_features
2017-11-05 21:59:13.029137 7f23302e4700  5 mon.ceph03@2(peon) e14 apply_monmap_to_compatset_features
2017-11-05 21:59:13.029140 7f23302e4700 10 mon.ceph03@2(peon) e14 timecheck_finish
2017-11-05 21:59:13.029141 7f23302e4700 10 mon.ceph03@2(peon) e14 resend_routed_requests
2017-11-05 21:59:13.029144 7f23302e4700 10 mon.ceph03@2(peon) e14 register_cluster_logger
2017-11-05 21:59:13.256981 7f232c2dc700 10 _calc_signature seq 248515229 front_crc_ = 3785316099 middle_crc = 0 data_crc = 0 sig = 15208736839717074848
2017-11-05 21:59:13.257038 7f232c2dc700 10 _calc_signature seq 248515230 front_crc_ = 2465577770 middle_crc = 0 data_crc = 0 sig = 4557865954043635196
2017-11-05 21:59:13.257066 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
2017-11-05 21:59:13.257090 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
2017-11-05 21:59:13.257098 7f23302e4700 20 is_capable service=mon command= read on cap allow *
2017-11-05 21:59:13.257101 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:13.257103 7f23302e4700 20  allow all
2017-11-05 21:59:13.257106 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
2017-11-05 21:59:13.257108 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:13.257110 7f23302e4700 20  allow all
2017-11-05 21:59:13.257115 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos recovering c 112907080..112907738) handle_collect paxos(collect lc 112907738 fc 112907080 pn 13700 opn 0) v4
2017-11-05 21:59:13.257126 7f23302e4700 20 mon.ceph03@2(peon).paxos(paxos recovering c 112907080..112907738) reset_lease_timeout - setting timeout event
2017-11-05 21:59:13.257133 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos recovering c 112907080..112907738) accepting pn 13700 from 0
2017-11-05 21:59:13.295049 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
2017-11-05 21:59:13.295058 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
2017-11-05 21:59:13.295061 7f23302e4700 20 is_capable service=mon command= read on cap allow *
2017-11-05 21:59:13.295064 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:13.295065 7f23302e4700 20  allow all
2017-11-05 21:59:13.295067 7f23302e4700 10 mon.ceph03@2(peon) e14 handle_timecheck time_check( ping e 344 r 1 ) v1
2017-11-05 21:59:13.295070 7f23302e4700 10 mon.ceph03@2(peon) e14 handle_timecheck_peon time_check( ping e 344 r 1 ) v1
2017-11-05 21:59:13.295065 7f232c2dc700 10 _calc_signature seq 7 front_crc_ = 769890768 middle_crc = 0 data_crc = 0 sig = 16174551769322836280
2017-11-05 21:59:13.295076 7f232c2dc700 20 Putting signature in client message(seq # 7): sig = 16174551769322836280
2017-11-05 21:59:13.295074 7f23302e4700 10 mon.ceph03@2(peon) e14 handle_timecheck_peon send time_check( ping e 344 r 1 ) v1 to mon.0 10.10.1.21:6789/0
2017-11-05 21:59:13.295116 7f232c2dc700 10 _calc_signature seq 8 front_crc_ = 2310135063 middle_crc = 0 data_crc = 0 sig = 16753175160622485731
2017-11-05 21:59:13.295121 7f232c2dc700 20 Putting signature in client message(seq # 8): sig = 16753175160622485731
2017-11-05 21:59:13.295420 7f232c2dc700 10 _calc_signature seq 248515231 front_crc_ = 2060387759 middle_crc = 0 data_crc = 0 sig = 15970751659405569903
2017-11-05 21:59:13.295525 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
2017-11-05 21:59:13.295534 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
2017-11-05 21:59:13.295537 7f23302e4700 20 is_capable service=mon command= read on cap allow *
2017-11-05 21:59:13.295540 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:13.295541 7f23302e4700 20  allow all
2017-11-05 21:59:13.295543 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
2017-11-05 21:59:13.295545 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:13.295546 7f23302e4700 20  allow all
2017-11-05 21:59:13.295551 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907738) handle_lease on 112907738 now 2017-11-05 21:59:18.293587
2017-11-05 21:59:13.295580 7f23302e4700 20 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907738) reset_lease_timeout - setting timeout event
2017-11-05 21:59:13.295589 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(pgmap 56600067..56600783) _active
2017-11-05 21:59:13.295591 7f23302e4700  7 mon.ceph03@2(peon).paxosservice(pgmap 56600067..56600783) _active we are not the leader, hence we propose nothing!
2017-11-05 21:59:13.295595 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mdsmap 1..1) _active
2017-11-05 21:59:13.295597 7f23302e4700  7 mon.ceph03@2(peon).paxosservice(mdsmap 1..1) _active we are not the leader, hence we propose nothing!
2017-11-05 21:59:13.295600 7f23302e4700 10 mon.ceph03@2(peon).mds e1
2017-11-05 21:59:13.295604 7f23302e4700 10 mon.ceph03@2(peon).mds e1 update_logger
2017-11-05 21:59:13.295607 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(osdmap 503..1105) _active
2017-11-05 21:59:13.295609 7f23302e4700  7 mon.ceph03@2(peon).paxosservice(osdmap 503..1105) _active we are not the leader, hence we propose nothing!
2017-11-05 21:59:13.295611 7f23302e4700 10 mon.ceph03@2(peon).osd e1105 update_logger
2017-11-05 21:59:13.295616 7f23302e4700 10 mon.ceph03@2(peon).osd e1105 take_all_failures on 0 osds
2017-11-05 21:59:13.295614 7f232c2dc700 10 _calc_signature seq 9 front_crc_ = 3465704679 middle_crc = 0 data_crc = 0 sig = 14492575556736995119
2017-11-05 21:59:13.295618 7f23302e4700 10 mon.ceph03@2(peon).osd e1105 start_mapping canceling previous mapping_job 0x7f2342d4ae00
2017-11-05 21:59:13.295620 7f232c2dc700 20 Putting signature in client message(seq # 9): sig = 14492575556736995119
2017-11-05 21:59:13.295628 7f23302e4700 20 queue 0x7f2342dddc60 1 [0,512)
2017-11-05 21:59:13.295633 7f23302e4700 20 queue 0x7f2342dddc60 2 [0,128)
2017-11-05 21:59:13.295639 7f23302e4700 20 queue 0x7f2342dddc60 4 [0,128)
2017-11-05 21:59:13.295641 7f23302e4700 10 mon.ceph03@2(peon).osd e1105 start_mapping started mapping job 0x7f2342dddc60 at 2017-11-05 21:59:13.295621
2017-11-05 21:59:13.295646 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(logm 55596915..55597634) _active
2017-11-05 21:59:13.295648 7f23302e4700  7 mon.ceph03@2(peon).paxosservice(logm 55596915..55597634) _active we are not the leader, hence we propose nothing!
2017-11-05 21:59:13.295650 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(monmap 1..14) _active
2017-11-05 21:59:13.295651 7f23302e4700  7 mon.ceph03@2(peon).paxosservice(monmap 1..14) _active we are not the leader, hence we propose nothing!
2017-11-05 21:59:13.295653 7f23302e4700  5 mon.ceph03@2(peon).monmap v14 apply_mon_features wait for service to be writeable
2017-11-05 21:59:13.295656 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(auth 26252..26331) _active
2017-11-05 21:59:13.295657 7f23302e4700  7 mon.ceph03@2(peon).paxosservice(auth 26252..26331) _active we are not the leader, hence we propose nothing!
2017-11-05 21:59:13.295659 7f23302e4700 10 mon.ceph03@2(peon).auth v26331 AuthMonitor::on_active()
2017-11-05 21:59:13.295654 7f23312e6700 20 _process 0x7f2342dddc60 1 [0,512)
2017-11-05 21:59:13.295663 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgr 1..394) _active
2017-11-05 21:59:13.295664 7f23302e4700  7 mon.ceph03@2(peon).paxosservice(mgr 1..394) _active we are not the leader, hence we propose nothing!
2017-11-05 21:59:13.295666 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgrstat 56609513..56609633) _active
2017-11-05 21:59:13.295668 7f23302e4700  7 mon.ceph03@2(peon).paxosservice(mgrstat 56609513..56609633) _active we are not the leader, hence we propose nothing!
2017-11-05 21:59:13.295670 7f23302e4700 20 mon.ceph03@2(peon).mgrstat update_logger
2017-11-05 21:59:13.295674 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(health 1..23) _active
2017-11-05 21:59:13.295676 7f23302e4700  7 mon.ceph03@2(peon).paxosservice(health 1..23) _active we are not the leader, hence we propose nothing!
2017-11-05 21:59:13.295678 7f23302e4700  5 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907738) is_readable = 1 - now=2017-11-05 21:59:13.295679 lease_expire=2017-11-05 21:59:18.293587 has v0 lc 112907738
2017-11-05 21:59:13.295686 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(logm 55596915..55597634) dispatch 0x7f234296e300 log(1 entries from seq 1 at 2017-11-05 21:59:12.841468) v1 from mon.2 10.10.1.23:6789/0 con 0x7f23431d6000
2017-11-05 21:59:13.295685 7f2331ae7700 20 _process 0x7f2342dddc60 2 [0,128)
2017-11-05 21:59:13.295694 7f23302e4700  5 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907738) is_readable = 1 - now=2017-11-05 21:59:13.295694 lease_expire=2017-11-05 21:59:18.293587 has v0 lc 112907738
2017-11-05 21:59:13.295695 7f2330ae5700 20 _process 0x7f2342dddc60 4 [0,128)
2017-11-05 21:59:13.295700 7f23302e4700 10 mon.ceph03@2(peon).log v55597634 preprocess_query log(1 entries from seq 1 at 2017-11-05 21:59:12.841468) v1 from mon.2 10.10.1.23:6789/0
2017-11-05 21:59:13.295706 7f23302e4700 10 mon.ceph03@2(peon).log v55597634 preprocess_log log(1 entries from seq 1 at 2017-11-05 21:59:12.841468) v1 from mon.2
2017-11-05 21:59:13.295711 7f23302e4700 20 is_capable service=log command= write on cap allow *
2017-11-05 21:59:13.295713 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:13.295714 7f23302e4700 20  allow all
2017-11-05 21:59:13.295732 7f23302e4700 10 mon.ceph03@2(peon) e14 forward_request 1 request log(1 entries from seq 1 at 2017-11-05 21:59:12.841468) v1 features 2305244844532236283
2017-11-05 21:59:13.295799 7f232c2dc700 10 _calc_signature seq 10 front_crc_ = 2470816670 middle_crc = 0 data_crc = 0 sig = 2328889181983519027
2017-11-05 21:59:13.295804 7f232c2dc700 20 Putting signature in client message(seq # 10): sig = 2328889181983519027
2017-11-05 21:59:13.297490 7f23312e6700 10 osdmap epoch 1105 mapping took 0.001868 seconds
2017-11-05 21:59:13.297500 7f23312e6700 10 mon.ceph03@2(peon).osd e1105 update_creating_pgs 0 pgs creating, 0 pools in queue
2017-11-05 21:59:13.440996 7f232c2dc700 10 _calc_signature seq 248515232 front_crc_ = 3060764253 middle_crc = 0 data_crc = 0 sig = 1336745259145746491
2017-11-05 21:59:13.441058 7f232c2dc700 10 _calc_signature seq 248515233 front_crc_ = 2324920290 middle_crc = 0 data_crc = 0 sig = 17879750372088187599
2017-11-05 21:59:13.441121 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
2017-11-05 21:59:13.441131 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
2017-11-05 21:59:13.441134 7f23302e4700 20 is_capable service=mon command= read on cap allow *
2017-11-05 21:59:13.441137 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:13.441138 7f23302e4700 20  allow all
2017-11-05 21:59:13.441140 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
2017-11-05 21:59:13.441141 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:13.441142 7f23302e4700 20  allow all
2017-11-05 21:59:13.441148 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907738) handle_begin paxos(begin lc 112907738 fc 0 pn 13700 opn 0) v4
2017-11-05 21:59:13.441155 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos updating c 112907080..112907738) accepting value for 112907739 pn 13700
2017-11-05 21:59:13.483332 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
2017-11-05 21:59:13.483342 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
2017-11-05 21:59:13.483345 7f23302e4700 20 is_capable service=mon command= read on cap allow *
2017-11-05 21:59:13.483347 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:13.483349 7f23302e4700 20  allow all
2017-11-05 21:59:13.483351 7f23302e4700 10 mon.ceph03@2(peon) e14 handle_timecheck time_check( report e 344 r 2 #skews 3 #latencies 3 ) v1
2017-11-05 21:59:13.483355 7f23302e4700 10 mon.ceph03@2(peon) e14 handle_timecheck_peon time_check( report e 344 r 2 #skews 3 #latencies 3 ) v1
2017-11-05 21:59:13.483370 7f232c2dc700 10 _calc_signature seq 11 front_crc_ = 4281357303 middle_crc = 0 data_crc = 0 sig = 5973436363658062779
2017-11-05 21:59:13.483381 7f232c2dc700 20 Putting signature in client message(seq # 11): sig = 5973436363658062779
2017-11-05 21:59:13.518562 7f232c2dc700 10 _calc_signature seq 248515234 front_crc_ = 2766364624 middle_crc = 0 data_crc = 0 sig = 1303804729337060943
2017-11-05 21:59:13.518675 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
2017-11-05 21:59:13.518684 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
2017-11-05 21:59:13.518687 7f23302e4700 20 is_capable service=mon command= read on cap allow *
2017-11-05 21:59:13.518696 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:13.518697 7f23302e4700 20  allow all
2017-11-05 21:59:13.518699 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
2017-11-05 21:59:13.518700 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:13.518701 7f23302e4700 20  allow all
2017-11-05 21:59:13.518708 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos updating c 112907080..112907738) handle_commit on 112907739
2017-11-05 21:59:13.518714 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos updating c 112907080..112907739) store_state [112907739..112907739]
2017-11-05 21:59:13.518793 7f232c2dc700 10 _calc_signature seq 248515235 front_crc_ = 2163361469 middle_crc = 0 data_crc = 0 sig = 2435996645971876188
2017-11-05 21:59:13.560989 7f23302e4700 10 mon.ceph03@2(peon) e14 refresh_from_paxos
2017-11-05 21:59:13.561022 7f23302e4700 10 mon.ceph03@2(peon) e14 refresh_from_paxos no cluster_fingerprint
2017-11-05 21:59:13.561053 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(pgmap 56600067..56600783) refresh
2017-11-05 21:59:13.561094 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mdsmap 1..1) refresh
2017-11-05 21:59:13.561139 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(osdmap 503..1105) refresh
2017-11-05 21:59:13.561172 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(logm 55596915..55597634) refresh
2017-11-05 21:59:13.561179 7f23302e4700 10 mon.ceph03@2(peon).log v55597634 update_from_paxos
2017-11-05 21:59:13.561181 7f23302e4700 10 mon.ceph03@2(peon).log v55597634 update_from_paxos version 55597634 summary v 55597634
2017-11-05 21:59:13.561220 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(monmap 1..14) refresh
2017-11-05 21:59:13.561250 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(auth 26252..26331) refresh
2017-11-05 21:59:13.561257 7f23302e4700 10 mon.ceph03@2(peon).auth v26331 update_from_paxos
2017-11-05 21:59:13.561295 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgr 1..394) refresh
2017-11-05 21:59:13.561349 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgrstat 56609513..56609633) refresh
2017-11-05 21:59:13.561355 7f23302e4700 10 mon.ceph03@2(peon).mgrstat  56609633
2017-11-05 21:59:13.561397 7f23302e4700 10 mon.ceph03@2(peon).mgrstat update_from_paxos v56609633 service_map e1
2017-11-05 21:59:13.561405 7f23302e4700 10 mon.ceph03@2(peon).mgrstat check_subs
2017-11-05 21:59:13.561407 7f23302e4700 20 mon.ceph03@2(peon).mgrstat update_logger
2017-11-05 21:59:13.561436 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(health 1..23) refresh
2017-11-05 21:59:13.561443 7f23302e4700 10 mon.ceph03@2(peon).health update_from_paxos
2017-11-05 21:59:13.561462 7f23302e4700 20 mon.ceph03@2(peon).health dump:{
    "quorum_health": {},
    "leader_health": {
        "MON_DOWN": {
            "severity": "HEALTH_WARN",
            "summary": {
                "message": "1/3 mons down, quorum ceph01mon,ceph02mon"
            },
            "detail": [
                {
                    "message": "mon.ceph03 (rank 2) addr 10.10.1.23:6789/0 is down (out of quorum)"
                }
            ]
        }
    }
}

2017-11-05 21:59:13.561489 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(pgmap 56600067..56600783) post_refresh
2017-11-05 21:59:13.561491 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mdsmap 1..1) post_refresh
2017-11-05 21:59:13.561493 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(osdmap 503..1105) post_refresh
2017-11-05 21:59:13.561495 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(logm 55596915..55597634) post_refresh
2017-11-05 21:59:13.561496 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(monmap 1..14) post_refresh
2017-11-05 21:59:13.561498 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(auth 26252..26331) post_refresh
2017-11-05 21:59:13.561499 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgr 1..394) post_refresh
2017-11-05 21:59:13.561501 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgrstat 56609513..56609633) post_refresh
2017-11-05 21:59:13.561502 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(health 1..23) post_refresh
2017-11-05 21:59:13.561597 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
2017-11-05 21:59:13.561606 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
2017-11-05 21:59:13.561609 7f23302e4700 20 is_capable service=mon command= read on cap allow *
2017-11-05 21:59:13.561611 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:13.561612 7f23302e4700 20  allow all
2017-11-05 21:59:13.561615 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
2017-11-05 21:59:13.561616 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:13.561617 7f23302e4700 20  allow all
2017-11-05 21:59:13.561622 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907739) handle_lease on 112907739 now 2017-11-05 21:59:18.516972
2017-11-05 21:59:13.561650 7f23302e4700 20 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907739) reset_lease_timeout - setting timeout event
2017-11-05 21:59:13.561656 7f23302e4700  5 mon.ceph03@2(peon).monmap v14 apply_mon_features wait for service to be writeable
2017-11-05 21:59:13.561660 7f23302e4700  5 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907739) is_readable = 1 - now=2017-11-05 21:59:13.561660 lease_expire=2017-11-05 21:59:18.516972 has v0 lc 112907739
2017-11-05 21:59:13.561722 7f232c2dc700 10 _calc_signature seq 12 front_crc_ = 2557579026 middle_crc = 0 data_crc = 0 sig = 17107270021298545528
2017-11-05 21:59:13.561742 7f232c2dc700 20 Putting signature in client message(seq # 12): sig = 17107270021298545528
2017-11-05 21:59:13.706944 7f232c2dc700 10 _calc_signature seq 248515236 front_crc_ = 2950603876 middle_crc = 0 data_crc = 0 sig = 14536828753144213213
2017-11-05 21:59:13.707035 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
2017-11-05 21:59:13.707045 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
2017-11-05 21:59:13.707048 7f23302e4700 20 is_capable service=mon command= read on cap allow *
2017-11-05 21:59:13.707051 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:13.707052 7f23302e4700 20  allow all
2017-11-05 21:59:13.707054 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
2017-11-05 21:59:13.707056 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:13.707062 7f23302e4700 20  allow all
2017-11-05 21:59:13.707069 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907739) handle_begin paxos(begin lc 112907739 fc 0 pn 13700 opn 0) v4
2017-11-05 21:59:13.707074 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos updating c 112907080..112907739) accepting value for 112907740 pn 13700
2017-11-05 21:59:13.749323 7f232c2dc700 10 _calc_signature seq 13 front_crc_ = 1277342878 middle_crc = 0 data_crc = 0 sig = 9029584966831276109
2017-11-05 21:59:13.749341 7f232c2dc700 20 Putting signature in client message(seq # 13): sig = 9029584966831276109
2017-11-05 21:59:13.785280 7f232c2dc700 10 _calc_signature seq 248515237 front_crc_ = 893193776 middle_crc = 0 data_crc = 0 sig = 2612573258296941357
2017-11-05 21:59:13.785376 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
2017-11-05 21:59:13.785385 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
2017-11-05 21:59:13.785388 7f23302e4700 20 is_capable service=mon command= read on cap allow *
2017-11-05 21:59:13.785391 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:13.785393 7f23302e4700 20  allow all
2017-11-05 21:59:13.785394 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
2017-11-05 21:59:13.785396 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:13.785397 7f23302e4700 20  allow all
2017-11-05 21:59:13.785403 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos updating c 112907080..112907739) handle_commit on 112907740
2017-11-05 21:59:13.785408 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos updating c 112907080..112907740) store_state [112907740..112907740]
2017-11-05 21:59:13.785510 7f232c2dc700 10 _calc_signature seq 248515238 front_crc_ = 4167574024 middle_crc = 0 data_crc = 0 sig = 4809870230413750984
2017-11-05 21:59:13.818516 7f232c2dc700 10 _calc_signature seq 248515239 front_crc_ = 1733102379 middle_crc = 0 data_crc = 0 sig = 9363108003920362173
2017-11-05 21:59:13.826915 7f23302e4700 10 mon.ceph03@2(peon) e14 refresh_from_paxos
2017-11-05 21:59:13.826944 7f23302e4700 10 mon.ceph03@2(peon) e14 refresh_from_paxos no cluster_fingerprint
2017-11-05 21:59:13.826972 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(pgmap 56600067..56600783) refresh
2017-11-05 21:59:13.827012 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mdsmap 1..1) refresh
2017-11-05 21:59:13.827051 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(osdmap 503..1105) refresh
2017-11-05 21:59:13.827082 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(logm 55596915..55597634) refresh
2017-11-05 21:59:13.827088 7f23302e4700 10 mon.ceph03@2(peon).log v55597634 update_from_paxos
2017-11-05 21:59:13.827091 7f23302e4700 10 mon.ceph03@2(peon).log v55597634 update_from_paxos version 55597634 summary v 55597634
2017-11-05 21:59:13.827124 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(monmap 1..14) refresh
2017-11-05 21:59:13.827151 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(auth 26252..26331) refresh
2017-11-05 21:59:13.827158 7f23302e4700 10 mon.ceph03@2(peon).auth v26331 update_from_paxos
2017-11-05 21:59:13.827192 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgr 1..394) refresh
2017-11-05 21:59:13.827239 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgrstat 56609513..56609634) refresh
2017-11-05 21:59:13.827249 7f23302e4700 10 mon.ceph03@2(peon).mgrstat  56609634
2017-11-05 21:59:13.827288 7f23302e4700 10 mon.ceph03@2(peon).mgrstat update_from_paxos v56609634 service_map e1
2017-11-05 21:59:13.827293 7f23302e4700 10 mon.ceph03@2(peon).mgrstat check_subs
2017-11-05 21:59:13.827295 7f23302e4700 20 mon.ceph03@2(peon).mgrstat update_logger
2017-11-05 21:59:13.827322 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(health 1..23) refresh
2017-11-05 21:59:13.827328 7f23302e4700 10 mon.ceph03@2(peon).health update_from_paxos
2017-11-05 21:59:13.827349 7f23302e4700 20 mon.ceph03@2(peon).health dump:{
    "quorum_health": {},
    "leader_health": {
        "MON_DOWN": {
            "severity": "HEALTH_WARN",
            "summary": {
                "message": "1/3 mons down, quorum ceph01mon,ceph02mon"
            },
            "detail": [
                {
                    "message": "mon.ceph03 (rank 2) addr 10.10.1.23:6789/0 is down (out of quorum)"
                }
            ]
        }
    }
}

2017-11-05 21:59:13.827372 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(pgmap 56600067..56600783) post_refresh
2017-11-05 21:59:13.827375 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mdsmap 1..1) post_refresh
2017-11-05 21:59:13.827377 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(osdmap 503..1105) post_refresh
2017-11-05 21:59:13.827378 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(logm 55596915..55597634) post_refresh
2017-11-05 21:59:13.827380 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(monmap 1..14) post_refresh
2017-11-05 21:59:13.827381 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(auth 26252..26331) post_refresh
2017-11-05 21:59:13.827383 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgr 1..394) post_refresh
2017-11-05 21:59:13.827384 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgrstat 56609513..56609634) post_refresh
2017-11-05 21:59:13.827385 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(health 1..23) post_refresh
2017-11-05 21:59:13.827472 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
2017-11-05 21:59:13.827479 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
2017-11-05 21:59:13.827482 7f23302e4700 20 is_capable service=mon command= read on cap allow *
2017-11-05 21:59:13.827484 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:13.827485 7f23302e4700 20  allow all
2017-11-05 21:59:13.827487 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
2017-11-05 21:59:13.827489 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:13.827490 7f23302e4700 20  allow all
2017-11-05 21:59:13.827494 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907740) handle_lease on 112907740 now 2017-11-05 21:59:18.783147
2017-11-05 21:59:13.827525 7f23302e4700 20 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907740) reset_lease_timeout - setting timeout event
2017-11-05 21:59:13.827532 7f23302e4700  5 mon.ceph03@2(peon).monmap v14 apply_mon_features wait for service to be writeable
2017-11-05 21:59:13.827536 7f23302e4700  5 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907740) is_readable = 1 - now=2017-11-05 21:59:13.827537 lease_expire=2017-11-05 21:59:18.783147 has v0 lc 112907740
2017-11-05 21:59:13.827563 7f232c2dc700 10 _calc_signature seq 14 front_crc_ = 1046939931 middle_crc = 0 data_crc = 0 sig = 137334039882502561
2017-11-05 21:59:13.827583 7f232c2dc700 20 Putting signature in client message(seq # 14): sig = 137334039882502561
2017-11-05 21:59:13.827566 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
2017-11-05 21:59:13.827590 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
2017-11-05 21:59:13.827601 7f23302e4700 20 is_capable service=mon command= read on cap allow *
2017-11-05 21:59:13.827603 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:13.827606 7f23302e4700 20  allow all
2017-11-05 21:59:13.827607 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
2017-11-05 21:59:13.827609 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:13.827610 7f23302e4700 20  allow all
2017-11-05 21:59:13.827619 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907740) handle_begin paxos(begin lc 112907740 fc 0 pn 13700 opn 0) v4
2017-11-05 21:59:13.827623 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos updating c 112907080..112907740) accepting value for 112907741 pn 13700
2017-11-05 21:59:13.871338 7f232c2dc700 10 _calc_signature seq 15 front_crc_ = 3677845416 middle_crc = 0 data_crc = 0 sig = 4007921578359993347
2017-11-05 21:59:13.871352 7f232c2dc700 20 Putting signature in client message(seq # 15): sig = 4007921578359993347
2017-11-05 21:59:13.907279 7f232c2dc700 10 _calc_signature seq 248515240 front_crc_ = 2867548840 middle_crc = 0 data_crc = 0 sig = 15577785751050191599
2017-11-05 21:59:13.907373 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
2017-11-05 21:59:13.907381 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
2017-11-05 21:59:13.907384 7f23302e4700 20 is_capable service=mon command= read on cap allow *
2017-11-05 21:59:13.907388 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:13.907390 7f23302e4700 20  allow all
2017-11-05 21:59:13.907391 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
2017-11-05 21:59:13.907392 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:13.907393 7f23302e4700 20  allow all
2017-11-05 21:59:13.907401 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos updating c 112907080..112907740) handle_commit on 112907741
2017-11-05 21:59:13.907405 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos updating c 112907080..112907741) store_state [112907741..112907741]
2017-11-05 21:59:13.907657 7f232c2dc700 10 _calc_signature seq 248515241 front_crc_ = 3270755471 middle_crc = 0 data_crc = 0 sig = 3345080007523614143
2017-11-05 21:59:13.907848 7f232c2dc700 10 _calc_signature seq 248515242 front_crc_ = 1530611789 middle_crc = 0 data_crc = 0 sig = 10373082544299377680
2017-11-05 21:59:13.949109 7f23302e4700 10 mon.ceph03@2(peon) e14 refresh_from_paxos
2017-11-05 21:59:13.949140 7f23302e4700 10 mon.ceph03@2(peon) e14 refresh_from_paxos no cluster_fingerprint
2017-11-05 21:59:13.949169 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(pgmap 56600067..56600783) refresh
2017-11-05 21:59:13.949206 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mdsmap 1..1) refresh
2017-11-05 21:59:13.949256 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(osdmap 503..1105) refresh
2017-11-05 21:59:13.949285 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(logm 55596915..55597635) refresh
2017-11-05 21:59:13.949290 7f23302e4700 10 mon.ceph03@2(peon).log v55597635 update_from_paxos
2017-11-05 21:59:13.949293 7f23302e4700 10 mon.ceph03@2(peon).log v55597635 update_from_paxos version 55597635 summary v 55597634
2017-11-05 21:59:13.949300 7f23302e4700 10 mon.ceph03@2(peon).log v55597635 update_from_paxos latest full 55597634
2017-11-05 21:59:13.949315 7f23302e4700  7 mon.ceph03@2(peon).log v55597635 update_from_paxos applying incremental log 55597635 2017-11-05 21:59:12.841468 mon.ceph03 mon.2 10.10.1.23:6789/0 1 : cluster [INF] mon.ceph03 calling new monitor election
2017-11-05 21:59:13.949335 7f23302e4700 20 mon.ceph03@2(peon).log v55597635 update_from_paxos logging for channel 'cluster' to file '/var/log/ceph/ceph.log'
2017-11-05 21:59:13.949351 7f23302e4700  7 mon.ceph03@2(peon).log v55597635 update_from_paxos applying incremental log 55597635 2017-11-05 21:59:12.884285 mon.ceph01mon mon.0 10.10.1.21:6789/0 855 : cluster [INF] mon.ceph01mon calling new monitor election
2017-11-05 21:59:13.949361 7f23302e4700 20 mon.ceph03@2(peon).log v55597635 update_from_paxos logging for channel 'cluster' to file '/var/log/ceph/ceph.log'
2017-11-05 21:59:13.949370 7f23302e4700  7 mon.ceph03@2(peon).log v55597635 update_from_paxos applying incremental log 55597635 2017-11-05 21:59:12.884938 mon.ceph02mon mon.1 10.10.1.22:6789/0 361 : cluster [INF] mon.ceph02mon calling new monitor election
2017-11-05 21:59:13.949376 7f23302e4700 20 mon.ceph03@2(peon).log v55597635 update_from_paxos logging for channel 'cluster' to file '/var/log/ceph/ceph.log'
2017-11-05 21:59:13.949384 7f23302e4700  7 mon.ceph03@2(peon).log v55597635 update_from_paxos applying incremental log 55597635 2017-11-05 21:59:12.984648 mon.ceph01mon mon.0 10.10.1.21:6789/0 856 : cluster [INF] mon.ceph01mon@0 won leader election with quorum 0,1,2
2017-11-05 21:59:13.949392 7f23302e4700 20 mon.ceph03@2(peon).log v55597635 update_from_paxos logging for channel 'cluster' to file '/var/log/ceph/ceph.log'
2017-11-05 21:59:13.949401 7f23302e4700  7 mon.ceph03@2(peon).log v55597635 update_from_paxos applying incremental log 55597635 2017-11-05 21:59:13.018039 mon.ceph01mon mon.0 10.10.1.21:6789/0 857 : cluster [WRN] overall HEALTH_WARN crush map has legacy tunables (require bobtail, min is firefly); nodeep-scrub flag(s) set; 1/3 mons down, quorum ceph01mon,ceph02mon
2017-11-05 21:59:13.949411 7f23302e4700 20 mon.ceph03@2(peon).log v55597635 update_from_paxos logging for channel 'cluster' to file '/var/log/ceph/ceph.log'
2017-11-05 21:59:13.949419 7f23302e4700  7 mon.ceph03@2(peon).log v55597635 update_from_paxos applying incremental log 55597635 2017-11-05 21:59:13.289046 mon.ceph01mon mon.0 10.10.1.21:6789/0 858 : cluster [INF] mon.1 10.10.1.22:6789/0
2017-11-05 21:59:13.949424 7f23302e4700 20 mon.ceph03@2(peon).log v55597635 update_from_paxos logging for channel 'cluster' to file '/var/log/ceph/ceph.log'
2017-11-05 21:59:13.949433 7f23302e4700  7 mon.ceph03@2(peon).log v55597635 update_from_paxos applying incremental log 55597635 2017-11-05 21:59:13.294021 mon.ceph01mon mon.0 10.10.1.21:6789/0 859 : cluster [INF] monmap e14: 3 mons at {ceph01mon=10.10.1.21:6789/0,ceph02mon=10.10.1.22:6789/0,ceph03=10.10.1.23:6789/0}
2017-11-05 21:59:13.949441 7f23302e4700 20 mon.ceph03@2(peon).log v55597635 update_from_paxos logging for channel 'cluster' to file '/var/log/ceph/ceph.log'
2017-11-05 21:59:13.949450 7f23302e4700  7 mon.ceph03@2(peon).log v55597635 update_from_paxos applying incremental log 55597635 2017-11-05 21:59:13.294068 mon.ceph01mon mon.0 10.10.1.21:6789/0 860 : cluster [DBG] fsmap
2017-11-05 21:59:13.949459 7f23302e4700 20 mon.ceph03@2(peon).log v55597635 update_from_paxos logging for channel 'cluster' to file '/var/log/ceph/ceph.log'
2017-11-05 21:59:13.949464 7f23302e4700  7 mon.ceph03@2(peon).log v55597635 update_from_paxos applying incremental log 55597635 2017-11-05 21:59:13.294143 mon.ceph01mon mon.0 10.10.1.21:6789/0 861 : cluster [DBG] osdmap e1105: 9 total, 9 up, 9 in
2017-11-05 21:59:13.949469 7f23302e4700 20 mon.ceph03@2(peon).log v55597635 update_from_paxos logging for channel 'cluster' to file '/var/log/ceph/ceph.log'
2017-11-05 21:59:13.949474 7f23302e4700  7 mon.ceph03@2(peon).log v55597635 update_from_paxos applying incremental log 55597635 2017-11-05 21:59:13.294231 mon.ceph01mon mon.0 10.10.1.21:6789/0 862 : cluster [DBG] mgrmap e394: ceph01mgr(active), standbys: ceph02mgr
2017-11-05 21:59:13.949479 7f23302e4700 20 mon.ceph03@2(peon).log v55597635 update_from_paxos logging for channel 'cluster' to file '/var/log/ceph/ceph.log'
2017-11-05 21:59:13.949486 7f23302e4700  7 mon.ceph03@2(peon).log v55597635 update_from_paxos applying incremental log 55597635 2017-11-05 21:59:13.438995 mon.ceph01mon mon.0 10.10.1.21:6789/0 863 : cluster [INF] mon.2 10.10.1.23:6789/0
2017-11-05 21:59:13.949494 7f23302e4700 20 mon.ceph03@2(peon).log v55597635 update_from_paxos logging for channel 'cluster' to file '/var/log/ceph/ceph.log'
2017-11-05 21:59:13.949513 7f23302e4700 15 mon.ceph03@2(peon).log v55597635 update_from_paxos logging for 1 channels
2017-11-05 21:59:13.949518 7f23302e4700 15 mon.ceph03@2(peon).log v55597635 update_from_paxos channel 'cluster' logging 1163 bytes
2017-11-05 21:59:13.949568 7f23302e4700 10 mon.ceph03@2(peon).log v55597635 check_subs
2017-11-05 21:59:13.949636 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(monmap 1..14) refresh
2017-11-05 21:59:13.949663 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(auth 26252..26331) refresh
2017-11-05 21:59:13.949669 7f23302e4700 10 mon.ceph03@2(peon).auth v26331 update_from_paxos
2017-11-05 21:59:13.949705 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgr 1..394) refresh
2017-11-05 21:59:13.949750 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgrstat 56609513..56609634) refresh
2017-11-05 21:59:13.949756 7f23302e4700 10 mon.ceph03@2(peon).mgrstat  56609634
2017-11-05 21:59:13.949791 7f23302e4700 10 mon.ceph03@2(peon).mgrstat update_from_paxos v56609634 service_map e1
2017-11-05 21:59:13.949796 7f23302e4700 10 mon.ceph03@2(peon).mgrstat check_subs
2017-11-05 21:59:13.949797 7f23302e4700 20 mon.ceph03@2(peon).mgrstat update_logger
2017-11-05 21:59:13.949822 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(health 1..23) refresh
2017-11-05 21:59:13.949828 7f23302e4700 10 mon.ceph03@2(peon).health update_from_paxos
2017-11-05 21:59:13.949849 7f23302e4700 20 mon.ceph03@2(peon).health dump:{
    "quorum_health": {},
    "leader_health": {
        "MON_DOWN": {
            "severity": "HEALTH_WARN",
            "summary": {
                "message": "1/3 mons down, quorum ceph01mon,ceph02mon"
            },
            "detail": [
                {
                    "message": "mon.ceph03 (rank 2) addr 10.10.1.23:6789/0 is down (out of quorum)"
                }
            ]
        }
    }
}

2017-11-05 21:59:13.949874 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(pgmap 56600067..56600783) post_refresh
2017-11-05 21:59:13.949876 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mdsmap 1..1) post_refresh
2017-11-05 21:59:13.949878 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(osdmap 503..1105) post_refresh
2017-11-05 21:59:13.949879 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(logm 55596915..55597635) post_refresh
2017-11-05 21:59:13.949881 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(monmap 1..14) post_refresh
2017-11-05 21:59:13.949882 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(auth 26252..26331) post_refresh
2017-11-05 21:59:13.949883 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgr 1..394) post_refresh
2017-11-05 21:59:13.949885 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgrstat 56609513..56609634) post_refresh
2017-11-05 21:59:13.949886 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(health 1..23) post_refresh
2017-11-05 21:59:13.949961 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
2017-11-05 21:59:13.949968 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
2017-11-05 21:59:13.949971 7f23302e4700 20 is_capable service=mon command= read on cap allow *
2017-11-05 21:59:13.949973 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:13.949974 7f23302e4700 20  allow all
2017-11-05 21:59:13.949975 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
2017-11-05 21:59:13.949979 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:13.949981 7f23302e4700 20  allow all
2017-11-05 21:59:13.949984 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907741) handle_lease on 112907741 now 2017-11-05 21:59:18.905672
2017-11-05 21:59:13.950008 7f23302e4700 20 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907741) reset_lease_timeout - setting timeout event
2017-11-05 21:59:13.950014 7f23302e4700  5 mon.ceph03@2(peon).monmap v14 apply_mon_features wait for service to be writeable
2017-11-05 21:59:13.950017 7f23302e4700  5 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907741) is_readable = 1 - now=2017-11-05 21:59:13.950017 lease_expire=2017-11-05 21:59:18.905672 has v0 lc 112907741
2017-11-05 21:59:13.950040 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
2017-11-05 21:59:13.950045 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
2017-11-05 21:59:13.950047 7f23302e4700 20 is_capable service=mon command= read on cap allow *
2017-11-05 21:59:13.950048 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:13.950050 7f23302e4700 20  allow all
2017-11-05 21:59:13.950053 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
2017-11-05 21:59:13.950054 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:13.950055 7f23302e4700 20  allow all
2017-11-05 21:59:13.950056 7f23302e4700 10 mon.ceph03@2(peon) e14 handle_route log(last 1) v1 to unknown.0 -
2017-11-05 21:59:13.950052 7f232c2dc700 10 _calc_signature seq 16 front_crc_ = 847570628 middle_crc = 0 data_crc = 0 sig = 12628439953592722512
2017-11-05 21:59:13.950062 7f232c2dc700 20 Putting signature in client message(seq # 16): sig = 12628439953592722512
2017-11-05 21:59:13.950109 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930a80 for mon.2 10.10.1.23:6789/0
2017-11-05 21:59:13.950116 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
2017-11-05 21:59:13.950118 7f23302e4700 20 is_capable service=mon command= read on cap allow *
2017-11-05 21:59:13.950119 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:13.950120 7f23302e4700 20  allow all
2017-11-05 21:59:14.937043 7f232c2dc700 10 _calc_signature seq 248515243 front_crc_ = 1914412222 middle_crc = 0 data_crc = 0 sig = 16176292074621219325
2017-11-05 21:59:14.937173 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
2017-11-05 21:59:14.937184 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
2017-11-05 21:59:14.937188 7f23302e4700 20 is_capable service=mon command= read on cap allow *
2017-11-05 21:59:14.937192 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:14.937193 7f23302e4700 20  allow all
2017-11-05 21:59:14.937195 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
2017-11-05 21:59:14.937210 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:14.937211 7f23302e4700 20  allow all
2017-11-05 21:59:14.937217 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907741) handle_begin paxos(begin lc 112907741 fc 0 pn 13700 opn 0) v4
2017-11-05 21:59:14.937222 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos updating c 112907080..112907741) accepting value for 112907742 pn 13700
2017-11-05 21:59:14.979132 7f232c2dc700 10 _calc_signature seq 17 front_crc_ = 87764062 middle_crc = 0 data_crc = 0 sig = 8597943680905522514
2017-11-05 21:59:14.979149 7f232c2dc700 20 Putting signature in client message(seq # 17): sig = 8597943680905522514
2017-11-05 21:59:15.014726 7f232c2dc700 10 _calc_signature seq 248515244 front_crc_ = 3809903813 middle_crc = 0 data_crc = 0 sig = 6400586654888549434
2017-11-05 21:59:15.014840 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
2017-11-05 21:59:15.014860 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
2017-11-05 21:59:15.014863 7f23302e4700 20 is_capable service=mon command= read on cap allow *
2017-11-05 21:59:15.014866 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:15.014867 7f23302e4700 20  allow all
2017-11-05 21:59:15.014869 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
2017-11-05 21:59:15.014871 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:15.014872 7f23302e4700 20  allow all
2017-11-05 21:59:15.014877 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos updating c 112907080..112907741) handle_commit on 112907742
2017-11-05 21:59:15.014882 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos updating c 112907080..112907742) store_state [112907742..112907742]
2017-11-05 21:59:15.015008 7f232c2dc700 10 _calc_signature seq 248515245 front_crc_ = 561089878 middle_crc = 0 data_crc = 0 sig = 17226791919200925804
2017-11-05 21:59:15.056780 7f23302e4700 10 mon.ceph03@2(peon) e14 refresh_from_paxos
2017-11-05 21:59:15.056816 7f23302e4700 10 mon.ceph03@2(peon) e14 refresh_from_paxos no cluster_fingerprint
2017-11-05 21:59:15.056852 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(pgmap 56600067..56600783) refresh
2017-11-05 21:59:15.056896 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mdsmap 1..1) refresh
2017-11-05 21:59:15.056944 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(osdmap 503..1105) refresh
2017-11-05 21:59:15.056978 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(logm 55596915..55597635) refresh
2017-11-05 21:59:15.056984 7f23302e4700 10 mon.ceph03@2(peon).log v55597635 update_from_paxos
2017-11-05 21:59:15.056987 7f23302e4700 10 mon.ceph03@2(peon).log v55597635 update_from_paxos version 55597635 summary v 55597635
2017-11-05 21:59:15.057031 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(monmap 1..14) refresh
2017-11-05 21:59:15.057063 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(auth 26252..26331) refresh
2017-11-05 21:59:15.057069 7f23302e4700 10 mon.ceph03@2(peon).auth v26331 update_from_paxos
2017-11-05 21:59:15.057111 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgr 1..394) refresh
2017-11-05 21:59:15.057175 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgrstat 56609513..56609635) refresh
2017-11-05 21:59:15.057183 7f23302e4700 10 mon.ceph03@2(peon).mgrstat  56609635
2017-11-05 21:59:15.057227 7f23302e4700 10 mon.ceph03@2(peon).mgrstat update_from_paxos v56609635 service_map e1
2017-11-05 21:59:15.057233 7f23302e4700 10 mon.ceph03@2(peon).mgrstat check_subs
2017-11-05 21:59:15.057235 7f23302e4700 20 mon.ceph03@2(peon).mgrstat update_logger
2017-11-05 21:59:15.057268 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(health 1..23) refresh
2017-11-05 21:59:15.057274 7f23302e4700 10 mon.ceph03@2(peon).health update_from_paxos
2017-11-05 21:59:15.057296 7f23302e4700 20 mon.ceph03@2(peon).health dump:{
    "quorum_health": {},
    "leader_health": {
        "MON_DOWN": {
            "severity": "HEALTH_WARN",
            "summary": {
                "message": "1/3 mons down, quorum ceph01mon,ceph02mon"
            },
            "detail": [
                {
                    "message": "mon.ceph03 (rank 2) addr 10.10.1.23:6789/0 is down (out of quorum)"
                }
            ]
        }
    }
}

2017-11-05 21:59:15.057322 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(pgmap 56600067..56600783) post_refresh
2017-11-05 21:59:15.057326 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mdsmap 1..1) post_refresh
2017-11-05 21:59:15.057328 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(osdmap 503..1105) post_refresh
2017-11-05 21:59:15.057330 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(logm 55596915..55597635) post_refresh
2017-11-05 21:59:15.057332 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(monmap 1..14) post_refresh
2017-11-05 21:59:15.057334 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(auth 26252..26331) post_refresh
2017-11-05 21:59:15.057335 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgr 1..394) post_refresh
2017-11-05 21:59:15.057337 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgrstat 56609513..56609635) post_refresh
2017-11-05 21:59:15.057339 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(health 1..23) post_refresh
2017-11-05 21:59:15.057427 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
2017-11-05 21:59:15.057435 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
2017-11-05 21:59:15.057438 7f23302e4700 20 is_capable service=mon command= read on cap allow *
2017-11-05 21:59:15.057441 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:15.057442 7f23302e4700 20  allow all
2017-11-05 21:59:15.057444 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
2017-11-05 21:59:15.057445 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:15.057447 7f23302e4700 20  allow all
2017-11-05 21:59:15.057451 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907742) handle_lease on 112907742 now 2017-11-05 21:59:20.013155
2017-11-05 21:59:15.057483 7f23302e4700 20 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907742) reset_lease_timeout - setting timeout event
2017-11-05 21:59:15.057490 7f23302e4700  5 mon.ceph03@2(peon).monmap v14 apply_mon_features wait for service to be writeable
2017-11-05 21:59:15.057494 7f23302e4700  5 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907742) is_readable = 1 - now=2017-11-05 21:59:15.057494 lease_expire=2017-11-05 21:59:20.013155 has v0 lc 112907742
2017-11-05 21:59:15.057553 7f232c2dc700 10 _calc_signature seq 18 front_crc_ = 3194447853 middle_crc = 0 data_crc = 0 sig = 17594015652951604420
2017-11-05 21:59:15.057571 7f232c2dc700 20 Putting signature in client message(seq # 18): sig = 17594015652951604420
2017-11-05 21:59:16.254989 7f232c2dc700 10 _calc_signature seq 248515246 front_crc_ = 3080749871 middle_crc = 0 data_crc = 0 sig = 13444557980616518344
2017-11-05 21:59:16.255107 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
2017-11-05 21:59:16.255118 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
2017-11-05 21:59:16.255122 7f23302e4700 20 is_capable service=mon command= read on cap allow *
2017-11-05 21:59:16.255126 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:16.255128 7f23302e4700 20  allow all
2017-11-05 21:59:16.255132 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
2017-11-05 21:59:16.255134 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:16.255135 7f23302e4700 20  allow all
2017-11-05 21:59:16.255142 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907742) handle_begin paxos(begin lc 112907742 fc 0 pn 13700 opn 0) v4
2017-11-05 21:59:16.255147 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos updating c 112907080..112907742) accepting value for 112907743 pn 13700
2017-11-05 21:59:16.296837 7f232c2dc700 10 _calc_signature seq 19 front_crc_ = 3209079344 middle_crc = 0 data_crc = 0 sig = 179244109618777490
2017-11-05 21:59:16.296861 7f232c2dc700 20 Putting signature in client message(seq # 19): sig = 179244109618777490
2017-11-05 21:59:16.332743 7f232c2dc700 10 _calc_signature seq 248515247 front_crc_ = 822332401 middle_crc = 0 data_crc = 0 sig = 15675063173283232976
2017-11-05 21:59:16.332852 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
2017-11-05 21:59:16.332861 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
2017-11-05 21:59:16.332864 7f23302e4700 20 is_capable service=mon command= read on cap allow *
2017-11-05 21:59:16.332867 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:16.332868 7f23302e4700 20  allow all
2017-11-05 21:59:16.332869 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
2017-11-05 21:59:16.332871 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:16.332872 7f23302e4700 20  allow all
2017-11-05 21:59:16.332876 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos updating c 112907080..112907742) handle_commit on 112907743
2017-11-05 21:59:16.332882 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos updating c 112907080..112907743) store_state [112907743..112907743]
2017-11-05 21:59:16.333041 7f232c2dc700 10 _calc_signature seq 248515248 front_crc_ = 3386463811 middle_crc = 0 data_crc = 0 sig = 14945621050292252831
2017-11-05 21:59:16.374435 7f23302e4700 10 mon.ceph03@2(peon) e14 refresh_from_paxos
2017-11-05 21:59:16.374470 7f23302e4700 10 mon.ceph03@2(peon) e14 refresh_from_paxos no cluster_fingerprint
2017-11-05 21:59:16.374504 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(pgmap 56600067..56600783) refresh
2017-11-05 21:59:16.374545 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mdsmap 1..1) refresh
2017-11-05 21:59:16.374590 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(osdmap 503..1105) refresh
2017-11-05 21:59:16.374623 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(logm 55596915..55597635) refresh
2017-11-05 21:59:16.374629 7f23302e4700 10 mon.ceph03@2(peon).log v55597635 update_from_paxos
2017-11-05 21:59:16.374631 7f23302e4700 10 mon.ceph03@2(peon).log v55597635 update_from_paxos version 55597635 summary v 55597635
2017-11-05 21:59:16.374670 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(monmap 1..14) refresh
2017-11-05 21:59:16.374699 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(auth 26252..26331) refresh
2017-11-05 21:59:16.374705 7f23302e4700 10 mon.ceph03@2(peon).auth v26331 update_from_paxos
2017-11-05 21:59:16.374744 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgr 1..394) refresh
2017-11-05 21:59:16.374798 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgrstat 56609513..56609636) refresh
2017-11-05 21:59:16.374804 7f23302e4700 10 mon.ceph03@2(peon).mgrstat  56609636
2017-11-05 21:59:16.374847 7f23302e4700 10 mon.ceph03@2(peon).mgrstat update_from_paxos v56609636 service_map e1
2017-11-05 21:59:16.374852 7f23302e4700 10 mon.ceph03@2(peon).mgrstat check_subs
2017-11-05 21:59:16.374854 7f23302e4700 20 mon.ceph03@2(peon).mgrstat update_logger
2017-11-05 21:59:16.374884 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(health 1..23) refresh
2017-11-05 21:59:16.374892 7f23302e4700 10 mon.ceph03@2(peon).health update_from_paxos
2017-11-05 21:59:16.374913 7f23302e4700 20 mon.ceph03@2(peon).health dump:{
    "quorum_health": {},
    "leader_health": {
        "MON_DOWN": {
            "severity": "HEALTH_WARN",
            "summary": {
                "message": "1/3 mons down, quorum ceph01mon,ceph02mon"
            },
            "detail": [
                {
                    "message": "mon.ceph03 (rank 2) addr 10.10.1.23:6789/0 is down (out of quorum)"
                }
            ]
        }
    }
}

2017-11-05 21:59:16.374935 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(pgmap 56600067..56600783) post_refresh
2017-11-05 21:59:16.374937 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mdsmap 1..1) post_refresh
2017-11-05 21:59:16.374939 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(osdmap 503..1105) post_refresh
2017-11-05 21:59:16.374940 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(logm 55596915..55597635) post_refresh
2017-11-05 21:59:16.374942 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(monmap 1..14) post_refresh
2017-11-05 21:59:16.374943 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(auth 26252..26331) post_refresh
2017-11-05 21:59:16.374945 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgr 1..394) post_refresh
2017-11-05 21:59:16.374946 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgrstat 56609513..56609636) post_refresh
2017-11-05 21:59:16.374949 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(health 1..23) post_refresh
2017-11-05 21:59:16.375031 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
2017-11-05 21:59:16.375041 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
2017-11-05 21:59:16.375044 7f23302e4700 20 is_capable service=mon command= read on cap allow *
2017-11-05 21:59:16.375047 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:16.375062 7f23302e4700 20  allow all
2017-11-05 21:59:16.375064 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
2017-11-05 21:59:16.375066 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:16.375067 7f23302e4700 20  allow all
2017-11-05 21:59:16.375072 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907743) handle_lease on 112907743 now 2017-11-05 21:59:21.331105
2017-11-05 21:59:16.375106 7f23302e4700 20 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907743) reset_lease_timeout - setting timeout event
2017-11-05 21:59:16.375116 7f23302e4700  5 mon.ceph03@2(peon).monmap v14 apply_mon_features wait for service to be writeable
2017-11-05 21:59:16.375120 7f23302e4700  5 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907743) is_readable = 1 - now=2017-11-05 21:59:16.375120 lease_expire=2017-11-05 21:59:21.331105 has v0 lc 112907743
2017-11-05 21:59:16.375160 7f232c2dc700 10 _calc_signature seq 20 front_crc_ = 3298296992 middle_crc = 0 data_crc = 0 sig = 17385314259900435840
2017-11-05 21:59:16.375177 7f232c2dc700 20 Putting signature in client message(seq # 20): sig = 17385314259900435840
2017-11-05 21:59:16.797649 7f232c2dc700 10 _calc_signature seq 248515249 front_crc_ = 1719413385 middle_crc = 0 data_crc = 0 sig = 6927146818612255408
2017-11-05 21:59:16.797744 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
2017-11-05 21:59:16.797754 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
2017-11-05 21:59:16.797758 7f23302e4700 20 is_capable service=mon command= read on cap allow *
2017-11-05 21:59:16.797761 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:16.797763 7f23302e4700 20  allow all
2017-11-05 21:59:16.797765 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
2017-11-05 21:59:16.797766 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:16.797767 7f23302e4700 20  allow all
2017-11-05 21:59:16.797773 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907743) handle_begin paxos(begin lc 112907743 fc 0 pn 13700 opn 0) v4
2017-11-05 21:59:16.797778 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos updating c 112907080..112907743) accepting value for 112907744 pn 13700
2017-11-05 21:59:16.839691 7f232c2dc700 10 _calc_signature seq 21 front_crc_ = 219823083 middle_crc = 0 data_crc = 0 sig = 16538990159248630615
2017-11-05 21:59:16.839710 7f232c2dc700 20 Putting signature in client message(seq # 21): sig = 16538990159248630615
2017-11-05 21:59:16.875406 7f232c2dc700 10 _calc_signature seq 248515250 front_crc_ = 3850737170 middle_crc = 0 data_crc = 0 sig = 17769774838374907603
2017-11-05 21:59:16.875492 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
2017-11-05 21:59:16.875501 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
2017-11-05 21:59:16.875504 7f23302e4700 20 is_capable service=mon command= read on cap allow *
2017-11-05 21:59:16.875506 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:16.875508 7f23302e4700 20  allow all
2017-11-05 21:59:16.875509 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
2017-11-05 21:59:16.875513 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:16.875514 7f23302e4700 20  allow all
2017-11-05 21:59:16.875518 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos updating c 112907080..112907743) handle_commit on 112907744
2017-11-05 21:59:16.875523 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos updating c 112907080..112907744) store_state [112907744..112907744]
2017-11-05 21:59:16.875701 7f232c2dc700 10 _calc_signature seq 248515251 front_crc_ = 2110551307 middle_crc = 0 data_crc = 0 sig = 6423051929449659683
2017-11-05 21:59:17.005793 7f23302e4700 10 mon.ceph03@2(peon) e14 refresh_from_paxos
2017-11-05 21:59:17.005828 7f23302e4700 10 mon.ceph03@2(peon) e14 refresh_from_paxos no cluster_fingerprint
2017-11-05 21:59:17.005862 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(pgmap 56600067..56600783) refresh
2017-11-05 21:59:17.005906 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mdsmap 1..1) refresh
2017-11-05 21:59:17.005952 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(osdmap 503..1105) refresh
2017-11-05 21:59:17.005984 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(logm 55596915..55597635) refresh
2017-11-05 21:59:17.005990 7f23302e4700 10 mon.ceph03@2(peon).log v55597635 update_from_paxos
2017-11-05 21:59:17.005993 7f23302e4700 10 mon.ceph03@2(peon).log v55597635 update_from_paxos version 55597635 summary v 55597635
2017-11-05 21:59:17.006031 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(monmap 1..14) refresh
2017-11-05 21:59:17.006060 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(auth 26252..26331) refresh
2017-11-05 21:59:17.006066 7f23302e4700 10 mon.ceph03@2(peon).auth v26331 update_from_paxos
2017-11-05 21:59:17.006105 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgr 1..394) refresh
2017-11-05 21:59:17.006160 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgrstat 56609513..56609636) refresh
2017-11-05 21:59:17.006168 7f23302e4700 10 mon.ceph03@2(peon).mgrstat  56609636
2017-11-05 21:59:17.006210 7f23302e4700 10 mon.ceph03@2(peon).mgrstat update_from_paxos v56609636 service_map e1
2017-11-05 21:59:17.006216 7f23302e4700 10 mon.ceph03@2(peon).mgrstat check_subs
2017-11-05 21:59:17.006218 7f23302e4700 20 mon.ceph03@2(peon).mgrstat update_logger
2017-11-05 21:59:17.006248 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(health 1..24) refresh
2017-11-05 21:59:17.006254 7f23302e4700 10 mon.ceph03@2(peon).health update_from_paxos
2017-11-05 21:59:17.006271 7f23302e4700 20 mon.ceph03@2(peon).health dump:{
    "quorum_health": {},
    "leader_health": {}
}

2017-11-05 21:59:17.006285 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(pgmap 56600067..56600783) post_refresh
2017-11-05 21:59:17.006288 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mdsmap 1..1) post_refresh
2017-11-05 21:59:17.006289 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(osdmap 503..1105) post_refresh
2017-11-05 21:59:17.006291 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(logm 55596915..55597635) post_refresh
2017-11-05 21:59:17.006292 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(monmap 1..14) post_refresh
2017-11-05 21:59:17.006294 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(auth 26252..26331) post_refresh
2017-11-05 21:59:17.006295 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgr 1..394) post_refresh
2017-11-05 21:59:17.006297 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgrstat 56609513..56609636) post_refresh
2017-11-05 21:59:17.006299 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(health 1..24) post_refresh
2017-11-05 21:59:17.006390 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
2017-11-05 21:59:17.006398 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
2017-11-05 21:59:17.006402 7f23302e4700 20 is_capable service=mon command= read on cap allow *
2017-11-05 21:59:17.006405 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:17.006406 7f23302e4700 20  allow all
2017-11-05 21:59:17.006408 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
2017-11-05 21:59:17.006409 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:17.006410 7f23302e4700 20  allow all
2017-11-05 21:59:17.006414 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907744) handle_lease on 112907744 now 2017-11-05 21:59:21.873858
2017-11-05 21:59:17.006444 7f23302e4700 20 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907744) reset_lease_timeout - setting timeout event
2017-11-05 21:59:17.006454 7f23302e4700  5 mon.ceph03@2(peon).monmap v14 apply_mon_features wait for service to be writeable
2017-11-05 21:59:17.006457 7f23302e4700  5 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907744) is_readable = 1 - now=2017-11-05 21:59:17.006458 lease_expire=2017-11-05 21:59:21.873858 has v0 lc 112907744
2017-11-05 21:59:17.006522 7f232c2dc700 10 _calc_signature seq 22 front_crc_ = 978136004 middle_crc = 0 data_crc = 0 sig = 7430388262177812061
2017-11-05 21:59:17.006540 7f232c2dc700 20 Putting signature in client message(seq # 22): sig = 7430388262177812061
2017-11-05 21:59:17.374028 7f232c2dc700 10 _calc_signature seq 248515252 front_crc_ = 2233000939 middle_crc = 0 data_crc = 0 sig = 5913206972756699187
2017-11-05 21:59:17.374152 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
2017-11-05 21:59:17.374163 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
2017-11-05 21:59:17.374167 7f23302e4700 20 is_capable service=mon command= read on cap allow *
2017-11-05 21:59:17.374170 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:17.374172 7f23302e4700 20  allow all
2017-11-05 21:59:17.374174 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
2017-11-05 21:59:17.374175 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:17.374176 7f23302e4700 20  allow all
2017-11-05 21:59:17.374182 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907744) handle_begin paxos(begin lc 112907744 fc 0 pn 13700 opn 0) v4
2017-11-05 21:59:17.374189 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos updating c 112907080..112907744) accepting value for 112907745 pn 13700
2017-11-05 21:59:17.415602 7f232c2dc700 10 _calc_signature seq 23 front_crc_ = 822198047 middle_crc = 0 data_crc = 0 sig = 11283865087405808684
2017-11-05 21:59:17.415619 7f232c2dc700 20 Putting signature in client message(seq # 23): sig = 11283865087405808684
2017-11-05 21:59:17.452463 7f232c2dc700 10 _calc_signature seq 248515253 front_crc_ = 4262535181 middle_crc = 0 data_crc = 0 sig = 9454139455262504054
2017-11-05 21:59:17.452517 7f232c2dc700 10 _calc_signature seq 248515254 front_crc_ = 3114647245 middle_crc = 0 data_crc = 0 sig = 1786097533032395933
2017-11-05 21:59:17.452572 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
2017-11-05 21:59:17.452582 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
2017-11-05 21:59:17.452585 7f23302e4700 20 is_capable service=mon command= read on cap allow *
2017-11-05 21:59:17.452588 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:17.452589 7f23302e4700 20  allow all
2017-11-05 21:59:17.452591 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
2017-11-05 21:59:17.452593 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:17.452594 7f23302e4700 20  allow all
2017-11-05 21:59:17.452599 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos updating c 112907080..112907744) handle_commit on 112907745
2017-11-05 21:59:17.452604 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos updating c 112907080..112907745) store_state [112907745..112907745]
2017-11-05 21:59:17.493220 7f23302e4700 10 mon.ceph03@2(peon) e14 refresh_from_paxos
2017-11-05 21:59:17.493254 7f23302e4700 10 mon.ceph03@2(peon) e14 refresh_from_paxos no cluster_fingerprint
2017-11-05 21:59:17.493288 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(pgmap 56600067..56600783) refresh
2017-11-05 21:59:17.493329 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mdsmap 1..1) refresh
2017-11-05 21:59:17.493374 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(osdmap 503..1105) refresh
2017-11-05 21:59:17.493407 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(logm 55596915..55597636) refresh
2017-11-05 21:59:17.493414 7f23302e4700 10 mon.ceph03@2(peon).log v55597636 update_from_paxos
2017-11-05 21:59:17.493417 7f23302e4700 10 mon.ceph03@2(peon).log v55597636 update_from_paxos version 55597636 summary v 55597635
2017-11-05 21:59:17.493425 7f23302e4700 10 mon.ceph03@2(peon).log v55597636 update_from_paxos latest full 55597635
2017-11-05 21:59:17.493441 7f23302e4700  7 mon.ceph03@2(peon).log v55597636 update_from_paxos applying incremental log 55597636 2017-11-05 21:59:16.764738 mon.ceph01mon mon.0 10.10.1.21:6789/0 864 : cluster [INF] Health check cleared: MON_DOWN (was: 1/3 mons down, quorum ceph01mon,ceph02mon)
2017-11-05 21:59:17.493463 7f23302e4700 20 mon.ceph03@2(peon).log v55597636 update_from_paxos logging for channel 'cluster' to file '/var/log/ceph/ceph.log'
2017-11-05 21:59:17.493490 7f23302e4700 15 mon.ceph03@2(peon).log v55597636 update_from_paxos logging for 1 channels
2017-11-05 21:59:17.493496 7f23302e4700 15 mon.ceph03@2(peon).log v55597636 update_from_paxos channel 'cluster' logging 165 bytes
2017-11-05 21:59:17.493551 7f23302e4700 10 mon.ceph03@2(peon).log v55597636 check_subs
2017-11-05 21:59:17.493604 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(monmap 1..14) refresh
2017-11-05 21:59:17.493635 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(auth 26252..26331) refresh
2017-11-05 21:59:17.493641 7f23302e4700 10 mon.ceph03@2(peon).auth v26331 update_from_paxos
2017-11-05 21:59:17.493681 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgr 1..394) refresh
2017-11-05 21:59:17.493734 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgrstat 56609513..56609636) refresh
2017-11-05 21:59:17.493740 7f23302e4700 10 mon.ceph03@2(peon).mgrstat  56609636
2017-11-05 21:59:17.493788 7f23302e4700 10 mon.ceph03@2(peon).mgrstat update_from_paxos v56609636 service_map e1
2017-11-05 21:59:17.493793 7f23302e4700 10 mon.ceph03@2(peon).mgrstat check_subs
2017-11-05 21:59:17.493795 7f23302e4700 20 mon.ceph03@2(peon).mgrstat update_logger
2017-11-05 21:59:17.493826 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(health 1..24) refresh
2017-11-05 21:59:17.493832 7f23302e4700 10 mon.ceph03@2(peon).health update_from_paxos
2017-11-05 21:59:17.493849 7f23302e4700 20 mon.ceph03@2(peon).health dump:{
    "quorum_health": {},
    "leader_health": {}
}

2017-11-05 21:59:17.493863 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(pgmap 56600067..56600783) post_refresh
2017-11-05 21:59:17.493866 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mdsmap 1..1) post_refresh
2017-11-05 21:59:17.493868 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(osdmap 503..1105) post_refresh
2017-11-05 21:59:17.493870 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(logm 55596915..55597636) post_refresh
2017-11-05 21:59:17.493872 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(monmap 1..14) post_refresh
2017-11-05 21:59:17.493874 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(auth 26252..26331) post_refresh
2017-11-05 21:59:17.493875 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgr 1..394) post_refresh
2017-11-05 21:59:17.493879 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgrstat 56609513..56609636) post_refresh
2017-11-05 21:59:17.493881 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(health 1..24) post_refresh
2017-11-05 21:59:17.493965 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
2017-11-05 21:59:17.493973 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
2017-11-05 21:59:17.493976 7f23302e4700 20 is_capable service=mon command= read on cap allow *
2017-11-05 21:59:17.493978 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:17.493980 7f23302e4700 20  allow all
2017-11-05 21:59:17.493981 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
2017-11-05 21:59:17.493983 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:17.493984 7f23302e4700 20  allow all
2017-11-05 21:59:17.493989 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907745) handle_lease on 112907745 now 2017-11-05 21:59:22.450233
2017-11-05 21:59:17.494014 7f23302e4700 20 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907745) reset_lease_timeout - setting timeout event
2017-11-05 21:59:17.494021 7f23302e4700  5 mon.ceph03@2(peon).monmap v14 apply_mon_features wait for service to be writeable
2017-11-05 21:59:17.494028 7f23302e4700  5 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907745) is_readable = 1 - now=2017-11-05 21:59:17.494029 lease_expire=2017-11-05 21:59:22.450233 has v0 lc 112907745
2017-11-05 21:59:17.494086 7f232c2dc700 10 _calc_signature seq 24 front_crc_ = 2877252748 middle_crc = 0 data_crc = 0 sig = 15957257717912452231
2017-11-05 21:59:17.494104 7f232c2dc700 20 Putting signature in client message(seq # 24): sig = 15957257717912452231
2017-11-05 21:59:17.691997 7f2332ae9700 11 mon.ceph03@2(peon) e14 tick
2017-11-05 21:59:17.692020 7f2332ae9700 10 mon.ceph03@2(peon).mds e1
2017-11-05 21:59:17.692026 7f2332ae9700 10 mon.ceph03@2(peon).osd e1105 e1105: 9 total, 9 up, 9 in
2017-11-05 21:59:17.692030 7f2332ae9700 10 mon.ceph03@2(peon).log v55597636 log
2017-11-05 21:59:17.692033 7f2332ae9700 10 mon.ceph03@2(peon).auth v26331 auth
2017-11-05 21:59:17.692036 7f2332ae9700 10 mon.ceph03@2(peon).health tick
2017-11-05 21:59:17.692038 7f2332ae9700 20 mon.ceph03@2(peon).health check_member_health
2017-11-05 21:59:17.692150 7f2332ae9700 10 mon.ceph03@2(peon).health check_member_health avail 92% total 277 GB, used 5676 MB, avail 257 GB
2017-11-05 21:59:17.692164 7f2332ae9700 20 mon.ceph03@2(peon) e14 sync_trim_providers
2017-11-05 21:59:17.898127 7f232d2de700 10 _calc_signature seq 3 front_crc_ = 1126695011 middle_crc = 0 data_crc = 0 sig = 13447063656664490716
2017-11-05 21:59:17.898145 7f232d2de700 20 Putting signature in client message(seq # 3): sig = 13447063656664490716
2017-11-05 21:59:18.481544 7f232c2dc700 10 _calc_signature seq 248515255 front_crc_ = 2335337029 middle_crc = 0 data_crc = 0 sig = 2844489569343748739
2017-11-05 21:59:18.481691 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
2017-11-05 21:59:18.481702 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
2017-11-05 21:59:18.481706 7f23302e4700 20 is_capable service=mon command= read on cap allow *
2017-11-05 21:59:18.481709 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:18.481710 7f23302e4700 20  allow all
2017-11-05 21:59:18.481713 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
2017-11-05 21:59:18.481714 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:18.481715 7f23302e4700 20  allow all
2017-11-05 21:59:18.481723 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907745) handle_begin paxos(begin lc 112907745 fc 0 pn 13700 opn 0) v4
2017-11-05 21:59:18.481728 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos updating c 112907080..112907745) accepting value for 112907746 pn 13700
2017-11-05 21:59:18.523045 7f232c2dc700 10 _calc_signature seq 25 front_crc_ = 4275636177 middle_crc = 0 data_crc = 0 sig = 14765727533992502905
2017-11-05 21:59:18.523060 7f232c2dc700 20 Putting signature in client message(seq # 25): sig = 14765727533992502905
2017-11-05 21:59:18.559375 7f232c2dc700 10 _calc_signature seq 248515256 front_crc_ = 2988239189 middle_crc = 0 data_crc = 0 sig = 4256597801313242484
2017-11-05 21:59:18.559489 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
2017-11-05 21:59:18.559498 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
2017-11-05 21:59:18.559502 7f23302e4700 20 is_capable service=mon command= read on cap allow *
2017-11-05 21:59:18.559505 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:18.559508 7f23302e4700 20  allow all
2017-11-05 21:59:18.559510 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
2017-11-05 21:59:18.559511 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:18.559513 7f23302e4700 20  allow all
2017-11-05 21:59:18.559518 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos updating c 112907080..112907745) handle_commit on 112907746
2017-11-05 21:59:18.559523 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos updating c 112907080..112907746) store_state [112907746..112907746]
2017-11-05 21:59:18.559666 7f232c2dc700 10 _calc_signature seq 248515257 front_crc_ = 1949826167 middle_crc = 0 data_crc = 0 sig = 1217847151241602786
2017-11-05 21:59:18.667111 7f23302e4700 10 mon.ceph03@2(peon) e14 refresh_from_paxos
2017-11-05 21:59:18.667147 7f23302e4700 10 mon.ceph03@2(peon) e14 refresh_from_paxos no cluster_fingerprint
2017-11-05 21:59:18.667185 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(pgmap 56600067..56600783) refresh
2017-11-05 21:59:18.667231 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mdsmap 1..1) refresh
2017-11-05 21:59:18.667280 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(osdmap 503..1105) refresh
2017-11-05 21:59:18.667315 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(logm 55596915..55597636) refresh
2017-11-05 21:59:18.667321 7f23302e4700 10 mon.ceph03@2(peon).log v55597636 update_from_paxos
2017-11-05 21:59:18.667326 7f23302e4700 10 mon.ceph03@2(peon).log v55597636 update_from_paxos version 55597636 summary v 55597636
2017-11-05 21:59:18.667367 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(monmap 1..14) refresh
2017-11-05 21:59:18.667399 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(auth 26252..26331) refresh
2017-11-05 21:59:18.667405 7f23302e4700 10 mon.ceph03@2(peon).auth v26331 update_from_paxos
2017-11-05 21:59:18.667448 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgr 1..394) refresh
2017-11-05 21:59:18.667515 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgrstat 56609513..56609637) refresh
2017-11-05 21:59:18.667523 7f23302e4700 10 mon.ceph03@2(peon).mgrstat  56609637
2017-11-05 21:59:18.667578 7f23302e4700 10 mon.ceph03@2(peon).mgrstat update_from_paxos v56609637 service_map e1
2017-11-05 21:59:18.667584 7f23302e4700 10 mon.ceph03@2(peon).mgrstat check_subs
2017-11-05 21:59:18.667587 7f23302e4700 20 mon.ceph03@2(peon).mgrstat update_logger
2017-11-05 21:59:18.667619 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(health 1..24) refresh
2017-11-05 21:59:18.667626 7f23302e4700 10 mon.ceph03@2(peon).health update_from_paxos
2017-11-05 21:59:18.667642 7f23302e4700 20 mon.ceph03@2(peon).health dump:{
    "quorum_health": {},
    "leader_health": {}
}

2017-11-05 21:59:18.667658 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(pgmap 56600067..56600783) post_refresh
2017-11-05 21:59:18.667661 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mdsmap 1..1) post_refresh
2017-11-05 21:59:18.667663 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(osdmap 503..1105) post_refresh
2017-11-05 21:59:18.667665 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(logm 55596915..55597636) post_refresh
2017-11-05 21:59:18.667667 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(monmap 1..14) post_refresh
2017-11-05 21:59:18.667669 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(auth 26252..26331) post_refresh
2017-11-05 21:59:18.667671 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgr 1..394) post_refresh
2017-11-05 21:59:18.667674 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgrstat 56609513..56609637) post_refresh
2017-11-05 21:59:18.667676 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(health 1..24) post_refresh
2017-11-05 21:59:18.667765 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
2017-11-05 21:59:18.667773 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
2017-11-05 21:59:18.667776 7f23302e4700 20 is_capable service=mon command= read on cap allow *
2017-11-05 21:59:18.667779 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:18.667780 7f23302e4700 20  allow all
2017-11-05 21:59:18.667782 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
2017-11-05 21:59:18.667784 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:18.667785 7f23302e4700 20  allow all
2017-11-05 21:59:18.667789 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907746) handle_lease on 112907746 now 2017-11-05 21:59:23.557741
2017-11-05 21:59:18.667821 7f23302e4700 20 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907746) reset_lease_timeout - setting timeout event
2017-11-05 21:59:18.667832 7f23302e4700  5 mon.ceph03@2(peon).monmap v14 apply_mon_features wait for service to be writeable
2017-11-05 21:59:18.667835 7f23302e4700  5 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907746) is_readable = 1 - now=2017-11-05 21:59:18.667836 lease_expire=2017-11-05 21:59:23.557741 has v0 lc 112907746
2017-11-05 21:59:18.667886 7f232c2dc700 10 _calc_signature seq 26 front_crc_ = 3644507818 middle_crc = 0 data_crc = 0 sig = 7016679905100796258
2017-11-05 21:59:18.667903 7f232c2dc700 20 Putting signature in client message(seq # 26): sig = 7016679905100796258
2017-11-05 21:59:20.253574 7f232c2dc700 10 _calc_signature seq 248515258 front_crc_ = 2845299607 middle_crc = 0 data_crc = 0 sig = 14165744638036507123
2017-11-05 21:59:20.253717 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
2017-11-05 21:59:20.253730 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
2017-11-05 21:59:20.253735 7f23302e4700 20 is_capable service=mon command= read on cap allow *
2017-11-05 21:59:20.253738 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:20.253740 7f23302e4700 20  allow all
2017-11-05 21:59:20.253742 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
2017-11-05 21:59:20.253744 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:20.253745 7f23302e4700 20  allow all
2017-11-05 21:59:20.253751 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907746) handle_begin paxos(begin lc 112907746 fc 0 pn 13700 opn 0) v4
2017-11-05 21:59:20.253756 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos updating c 112907080..112907746) accepting value for 112907747 pn 13700
2017-11-05 21:59:20.294606 7f232c2dc700 10 _calc_signature seq 27 front_crc_ = 2216739829 middle_crc = 0 data_crc = 0 sig = 14551920355953459298
2017-11-05 21:59:20.294651 7f232c2dc700 20 Putting signature in client message(seq # 27): sig = 14551920355953459298
2017-11-05 21:59:20.331167 7f232c2dc700 10 _calc_signature seq 248515259 front_crc_ = 2227490693 middle_crc = 0 data_crc = 0 sig = 2808689771223304806
2017-11-05 21:59:20.331308 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
2017-11-05 21:59:20.331318 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
2017-11-05 21:59:20.331326 7f23302e4700 20 is_capable service=mon command= read on cap allow *
2017-11-05 21:59:20.331328 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:20.331329 7f23302e4700 20  allow all
2017-11-05 21:59:20.331331 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
2017-11-05 21:59:20.331332 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:20.331333 7f23302e4700 20  allow all
2017-11-05 21:59:20.331338 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos updating c 112907080..112907746) handle_commit on 112907747
2017-11-05 21:59:20.331343 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos updating c 112907080..112907747) store_state [112907747..112907747]
2017-11-05 21:59:20.331448 7f232c2dc700 10 _calc_signature seq 248515260 front_crc_ = 618269879 middle_crc = 0 data_crc = 0 sig = 18220854632036809290
2017-11-05 21:59:20.372209 7f23302e4700 10 mon.ceph03@2(peon) e14 refresh_from_paxos
2017-11-05 21:59:20.372243 7f23302e4700 10 mon.ceph03@2(peon) e14 refresh_from_paxos no cluster_fingerprint
2017-11-05 21:59:20.372281 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(pgmap 56600067..56600783) refresh
2017-11-05 21:59:20.372332 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mdsmap 1..1) refresh
2017-11-05 21:59:20.372380 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(osdmap 503..1105) refresh
2017-11-05 21:59:20.372427 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(logm 55596915..55597636) refresh
2017-11-05 21:59:20.372433 7f23302e4700 10 mon.ceph03@2(peon).log v55597636 update_from_paxos
2017-11-05 21:59:20.372435 7f23302e4700 10 mon.ceph03@2(peon).log v55597636 update_from_paxos version 55597636 summary v 55597636
2017-11-05 21:59:20.372474 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(monmap 1..14) refresh
2017-11-05 21:59:20.372504 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(auth 26252..26331) refresh
2017-11-05 21:59:20.372510 7f23302e4700 10 mon.ceph03@2(peon).auth v26331 update_from_paxos
2017-11-05 21:59:20.372550 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgr 1..394) refresh
2017-11-05 21:59:20.372612 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgrstat 56609513..56609638) refresh
2017-11-05 21:59:20.372619 7f23302e4700 10 mon.ceph03@2(peon).mgrstat  56609638
2017-11-05 21:59:20.372666 7f23302e4700 10 mon.ceph03@2(peon).mgrstat update_from_paxos v56609638 service_map e1
2017-11-05 21:59:20.372672 7f23302e4700 10 mon.ceph03@2(peon).mgrstat check_subs
2017-11-05 21:59:20.372674 7f23302e4700 20 mon.ceph03@2(peon).mgrstat update_logger
2017-11-05 21:59:20.372704 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(health 1..24) refresh
2017-11-05 21:59:20.372711 7f23302e4700 10 mon.ceph03@2(peon).health update_from_paxos
2017-11-05 21:59:20.372726 7f23302e4700 20 mon.ceph03@2(peon).health dump:{
    "quorum_health": {},
    "leader_health": {}
}

2017-11-05 21:59:20.372741 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(pgmap 56600067..56600783) post_refresh
2017-11-05 21:59:20.372745 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mdsmap 1..1) post_refresh
2017-11-05 21:59:20.372747 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(osdmap 503..1105) post_refresh
2017-11-05 21:59:20.372748 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(logm 55596915..55597636) post_refresh
2017-11-05 21:59:20.372750 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(monmap 1..14) post_refresh
2017-11-05 21:59:20.372752 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(auth 26252..26331) post_refresh
2017-11-05 21:59:20.372753 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgr 1..394) post_refresh
2017-11-05 21:59:20.372755 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgrstat 56609513..56609638) post_refresh
2017-11-05 21:59:20.372756 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(health 1..24) post_refresh
2017-11-05 21:59:20.372846 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
2017-11-05 21:59:20.372854 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
2017-11-05 21:59:20.372857 7f23302e4700 20 is_capable service=mon command= read on cap allow *
2017-11-05 21:59:20.372859 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:20.372860 7f23302e4700 20  allow all
2017-11-05 21:59:20.372862 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
2017-11-05 21:59:20.372864 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:20.372866 7f23302e4700 20  allow all
2017-11-05 21:59:20.372870 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907747) handle_lease on 112907747 now 2017-11-05 21:59:25.329563
2017-11-05 21:59:20.372901 7f23302e4700 20 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907747) reset_lease_timeout - setting timeout event
2017-11-05 21:59:20.372908 7f23302e4700  5 mon.ceph03@2(peon).monmap v14 apply_mon_features wait for service to be writeable
2017-11-05 21:59:20.372915 7f23302e4700  5 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907747) is_readable = 1 - now=2017-11-05 21:59:20.372916 lease_expire=2017-11-05 21:59:25.329563 has v0 lc 112907747
2017-11-05 21:59:20.372967 7f232c2dc700 10 _calc_signature seq 28 front_crc_ = 3419319976 middle_crc = 0 data_crc = 0 sig = 17296926581295683541
2017-11-05 21:59:20.372984 7f232c2dc700 20 Putting signature in client message(seq # 28): sig = 17296926581295683541
2017-11-05 21:59:20.731044 7f232d2de700 10 mon.ceph03@2(peon) e14 ms_verify_authorizer 10.10.1.21:0/2729617137 client protocol 0
2017-11-05 21:59:20.731063 7f232d2de700 10 In get_auth_session_handler for protocol 0
2017-11-05 21:59:20.731347 7f23302e4700 10 mon.ceph03@2(peon) e14 _ms_dispatch new session 0x7f2342933180 MonSession(client.? 10.10.1.21:0/2729617137 is open ) features 0x1ffddff8eea4fffb
2017-11-05 21:59:20.731359 7f23302e4700 20 mon.ceph03@2(peon) e14  caps
2017-11-05 21:59:20.731363 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(auth 26252..26331) dispatch 0x7f234296d900 auth(proto 0 30 bytes epoch 0) v1 from client.? 10.10.1.21:0/2729617137 con 0x7f2342de6800
2017-11-05 21:59:20.731370 7f23302e4700  5 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907747) is_readable = 1 - now=2017-11-05 21:59:20.731371 lease_expire=2017-11-05 21:59:25.329563 has v0 lc 112907747
2017-11-05 21:59:20.731380 7f23302e4700 10 mon.ceph03@2(peon).auth v26331 preprocess_query auth(proto 0 30 bytes epoch 0) v1 from client.? 10.10.1.21:0/2729617137
2017-11-05 21:59:20.731386 7f23302e4700 10 mon.ceph03@2(peon).auth v26331 prep_auth() blob_size=30
2017-11-05 21:59:20.731404 7f23302e4700 10 mon.ceph03@2(peon).auth v26331 AuthMonitor::assign_global_id m=auth(proto 0 30 bytes epoch 0) v1 mon=2/3 last_allocated=55234096 max_global_id=55234096
2017-11-05 21:59:20.731412 7f23302e4700 10 mon.ceph03@2(peon).auth v26331 next_global_id should be 55234100
2017-11-05 21:59:20.731414 7f23302e4700 10 mon.ceph03@2(peon).auth v26331 not the leader, requesting more ids from leader
2017-11-05 21:59:20.731491 7f232c2dc700 10 _calc_signature seq 29 front_crc_ = 1755414638 middle_crc = 0 data_crc = 0 sig = 10624911126847986870
2017-11-05 21:59:20.731509 7f232c2dc700 20 Putting signature in client message(seq # 29): sig = 10624911126847986870
2017-11-05 21:59:20.732185 7f23302e4700 10 mon.ceph03@2(peon) e14 ms_handle_reset 0x7f2342de6800 10.10.1.21:0/2729617137
2017-11-05 21:59:20.732209 7f23302e4700 10 mon.ceph03@2(peon) e14 reset/close on session client.? 10.10.1.21:0/2729617137
2017-11-05 21:59:20.732214 7f23302e4700 10 mon.ceph03@2(peon) e14 remove_session 0x7f2342933180 client.? 10.10.1.21:0/2729617137 features 0x1ffddff8eea4fffb
2017-11-05 21:59:20.763005 7f232c2dc700 10 _calc_signature seq 248515261 front_crc_ = 2020420307 middle_crc = 0 data_crc = 0 sig = 16491983375567104385
2017-11-05 21:59:20.763112 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
2017-11-05 21:59:20.763121 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
2017-11-05 21:59:20.763125 7f23302e4700 20 is_capable service=mon command= read on cap allow *
2017-11-05 21:59:20.763128 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:20.763129 7f23302e4700 20  allow all
2017-11-05 21:59:20.763131 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
2017-11-05 21:59:20.763133 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:20.763134 7f23302e4700 20  allow all
2017-11-05 21:59:20.763139 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907747) handle_begin paxos(begin lc 112907747 fc 0 pn 13700 opn 0) v4
2017-11-05 21:59:20.763143 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos updating c 112907080..112907747) accepting value for 112907748 pn 13700
2017-11-05 21:59:20.804165 7f232c2dc700 10 _calc_signature seq 30 front_crc_ = 4077125949 middle_crc = 0 data_crc = 0 sig = 2958054525900925623
2017-11-05 21:59:20.804183 7f232c2dc700 20 Putting signature in client message(seq # 30): sig = 2958054525900925623
2017-11-05 21:59:20.840669 7f232c2dc700 10 _calc_signature seq 248515262 front_crc_ = 2514305494 middle_crc = 0 data_crc = 0 sig = 3250237394762425023
2017-11-05 21:59:20.840777 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
2017-11-05 21:59:20.840789 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
2017-11-05 21:59:20.840793 7f23302e4700 20 is_capable service=mon command= read on cap allow *
2017-11-05 21:59:20.840796 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:20.840797 7f23302e4700 20  allow all
2017-11-05 21:59:20.840799 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
2017-11-05 21:59:20.840801 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:20.840802 7f23302e4700 20  allow all
2017-11-05 21:59:20.840807 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos updating c 112907080..112907747) handle_commit on 112907748
2017-11-05 21:59:20.840831 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos updating c 112907080..112907748) store_state [112907748..112907748]
2017-11-05 21:59:20.840896 7f232c2dc700 10 _calc_signature seq 248515263 front_crc_ = 406934426 middle_crc = 0 data_crc = 0 sig = 15435441924216793349
2017-11-05 21:59:20.881717 7f23302e4700 10 mon.ceph03@2(peon) e14 refresh_from_paxos
2017-11-05 21:59:20.881751 7f23302e4700 10 mon.ceph03@2(peon) e14 refresh_from_paxos no cluster_fingerprint
2017-11-05 21:59:20.881787 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(pgmap 56600067..56600783) refresh
2017-11-05 21:59:20.881833 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mdsmap 1..1) refresh
2017-11-05 21:59:20.881879 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(osdmap 503..1105) refresh
2017-11-05 21:59:20.881912 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(logm 55596915..55597636) refresh
2017-11-05 21:59:20.881918 7f23302e4700 10 mon.ceph03@2(peon).log v55597636 update_from_paxos
2017-11-05 21:59:20.881921 7f23302e4700 10 mon.ceph03@2(peon).log v55597636 update_from_paxos version 55597636 summary v 55597636
2017-11-05 21:59:20.881960 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(monmap 1..14) refresh
2017-11-05 21:59:20.881984 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(auth 26252..26332) refresh
2017-11-05 21:59:20.881990 7f23302e4700 10 mon.ceph03@2(peon).auth v26332 update_from_paxos
2017-11-05 21:59:20.882006 7f23302e4700 10 mon.ceph03@2(peon).auth v26332 update_from_paxos version 26332 keys ver 26331 latest 26321
2017-11-05 21:59:20.882015 7f23302e4700 10 mon.ceph03@2(peon).auth v26332 update_from_paxos key server version 26331
2017-11-05 21:59:20.882024 7f23302e4700 20 mon.ceph03@2(peon).auth v26332 update_from_paxos walking through version 26332 len 19
2017-11-05 21:59:20.882028 7f23302e4700 10 mon.ceph03@2(peon).auth v26332 update_from_paxos() last_allocated_id=55234096 max_global_id=55244096 format_version 2
2017-11-05 21:59:20.882068 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgr 1..394) refresh
2017-11-05 21:59:20.882125 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgrstat 56609513..56609638) refresh
2017-11-05 21:59:20.882133 7f23302e4700 10 mon.ceph03@2(peon).mgrstat  56609638
2017-11-05 21:59:20.882181 7f23302e4700 10 mon.ceph03@2(peon).mgrstat update_from_paxos v56609638 service_map e1
2017-11-05 21:59:20.882187 7f23302e4700 10 mon.ceph03@2(peon).mgrstat check_subs
2017-11-05 21:59:20.882189 7f23302e4700 20 mon.ceph03@2(peon).mgrstat update_logger
2017-11-05 21:59:20.882221 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(health 1..24) refresh
2017-11-05 21:59:20.882228 7f23302e4700 10 mon.ceph03@2(peon).health update_from_paxos
2017-11-05 21:59:20.882244 7f23302e4700 20 mon.ceph03@2(peon).health dump:{
    "quorum_health": {},
    "leader_health": {}
}

2017-11-05 21:59:20.882259 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(pgmap 56600067..56600783) post_refresh
2017-11-05 21:59:20.882261 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mdsmap 1..1) post_refresh
2017-11-05 21:59:20.882263 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(osdmap 503..1105) post_refresh
2017-11-05 21:59:20.882265 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(logm 55596915..55597636) post_refresh
2017-11-05 21:59:20.882267 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(monmap 1..14) post_refresh
2017-11-05 21:59:20.882269 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(auth 26252..26332) post_refresh
2017-11-05 21:59:20.882273 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(auth 26252..26332) dispatch 0x7f234296d900 auth(proto 0 30 bytes epoch 0) v1 from client.? 10.10.1.21:0/2729617137 con 0x7f2342de6800
2017-11-05 21:59:20.882299 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(auth 26252..26332)  discarding message from disconnected client client.? 10.10.1.21:0/2729617137 auth(proto 0 30 bytes epoch 0) v1
2017-11-05 21:59:20.882321 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgr 1..394) post_refresh
2017-11-05 21:59:20.882327 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgrstat 56609513..56609638) post_refresh
2017-11-05 21:59:20.882329 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(health 1..24) post_refresh
2017-11-05 21:59:20.882417 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
2017-11-05 21:59:20.882425 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
2017-11-05 21:59:20.882429 7f23302e4700 20 is_capable service=mon command= read on cap allow *
2017-11-05 21:59:20.882431 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:20.882433 7f23302e4700 20  allow all
2017-11-05 21:59:20.882435 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
2017-11-05 21:59:20.882436 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:20.882437 7f23302e4700 20  allow all
2017-11-05 21:59:20.882441 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907748) handle_lease on 112907748 now 2017-11-05 21:59:25.839092
2017-11-05 21:59:20.882475 7f23302e4700 20 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907748) reset_lease_timeout - setting timeout event
2017-11-05 21:59:20.882481 7f23302e4700  5 mon.ceph03@2(peon).monmap v14 apply_mon_features wait for service to be writeable
2017-11-05 21:59:20.882484 7f23302e4700  5 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907748) is_readable = 1 - now=2017-11-05 21:59:20.882484 lease_expire=2017-11-05 21:59:25.839092 has v0 lc 112907748
2017-11-05 21:59:20.882541 7f232c2dc700 10 _calc_signature seq 31 front_crc_ = 235397502 middle_crc = 0 data_crc = 0 sig = 8528813682334572839
2017-11-05 21:59:20.882558 7f232c2dc700 20 Putting signature in client message(seq # 31): sig = 8528813682334572839
2017-11-05 21:59:21.870644 7f232c2dc700 10 _calc_signature seq 248515264 front_crc_ = 791506274 middle_crc = 0 data_crc = 0 sig = 13748375555303897324
2017-11-05 21:59:21.870752 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
2017-11-05 21:59:21.870763 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
2017-11-05 21:59:21.870767 7f23302e4700 20 is_capable service=mon command= read on cap allow *
2017-11-05 21:59:21.870771 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:21.870772 7f23302e4700 20  allow all
2017-11-05 21:59:21.870775 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
2017-11-05 21:59:21.870776 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:21.870778 7f23302e4700 20  allow all
2017-11-05 21:59:21.870785 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907748) handle_begin paxos(begin lc 112907748 fc 0 pn 13700 opn 0) v4
2017-11-05 21:59:21.870790 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos updating c 112907080..112907748) accepting value for 112907749 pn 13700
2017-11-05 21:59:21.911518 7f232c2dc700 10 _calc_signature seq 32 front_crc_ = 3333053517 middle_crc = 0 data_crc = 0 sig = 8416660674998655820
2017-11-05 21:59:21.911534 7f232c2dc700 20 Putting signature in client message(seq # 32): sig = 8416660674998655820
2017-11-05 21:59:21.949630 7f232c2dc700 10 _calc_signature seq 248515265 front_crc_ = 370334524 middle_crc = 0 data_crc = 0 sig = 5896854534053460549
2017-11-05 21:59:21.949717 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
2017-11-05 21:59:21.949727 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
2017-11-05 21:59:21.949733 7f23302e4700 20 is_capable service=mon command= read on cap allow *
2017-11-05 21:59:21.949736 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:21.949738 7f23302e4700 20  allow all
2017-11-05 21:59:21.949740 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
2017-11-05 21:59:21.949741 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:21.949744 7f23302e4700 20  allow all
2017-11-05 21:59:21.949769 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos updating c 112907080..112907748) handle_commit on 112907749
2017-11-05 21:59:21.949778 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos updating c 112907080..112907749) store_state [112907749..112907749]
2017-11-05 21:59:21.950003 7f232c2dc700 10 _calc_signature seq 248515266 front_crc_ = 2497589639 middle_crc = 0 data_crc = 0 sig = 8276221671505461764
2017-11-05 21:59:21.989093 7f23302e4700 10 mon.ceph03@2(peon) e14 refresh_from_paxos
2017-11-05 21:59:21.989125 7f23302e4700 10 mon.ceph03@2(peon) e14 refresh_from_paxos no cluster_fingerprint
2017-11-05 21:59:21.989158 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(pgmap 56600067..56600783) refresh
2017-11-05 21:59:21.989203 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mdsmap 1..1) refresh
2017-11-05 21:59:21.989246 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(osdmap 503..1105) refresh
2017-11-05 21:59:21.989277 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(logm 55596915..55597637) refresh
2017-11-05 21:59:21.989284 7f23302e4700 10 mon.ceph03@2(peon).log v55597637 update_from_paxos
2017-11-05 21:59:21.989286 7f23302e4700 10 mon.ceph03@2(peon).log v55597637 update_from_paxos version 55597637 summary v 55597636
2017-11-05 21:59:21.989294 7f23302e4700 10 mon.ceph03@2(peon).log v55597637 update_from_paxos latest full 55597636
2017-11-05 21:59:21.989312 7f23302e4700  7 mon.ceph03@2(peon).log v55597637 update_from_paxos applying incremental log 55597637 2017-11-05 21:59:20.718439 mgr.ceph01mgr client.54805773 10.10.1.21:0/1619113657 1818 : audit [DBG] from='client.55026405 10.10.1.21:0/3042078038' entity='client.admin' cmd=[{"prefix": "osd pool stats", "target": ["mgr", ""], "format": "json"}]: dispatch
2017-11-05 21:59:21.989333 7f23302e4700 20 mon.ceph03@2(peon).log v55597637 update_from_paxos logging for channel 'audit' to file '/var/log/ceph/ceph.audit.log'
2017-11-05 21:59:21.989342 7f23302e4700  7 mon.ceph03@2(peon).log v55597637 update_from_paxos applying incremental log 55597637 2017-11-05 21:59:20.943050 mon.ceph01mon mon.0 10.10.1.21:6789/0 865 : audit [DBG] from='client.? 10.10.1.21:0/2729617137' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch
2017-11-05 21:59:21.989352 7f23302e4700 20 mon.ceph03@2(peon).log v55597637 update_from_paxos logging for channel 'audit' to file '/var/log/ceph/ceph.audit.log'
2017-11-05 21:59:21.989358 7f23302e4700  7 mon.ceph03@2(peon).log v55597637 update_from_paxos applying incremental log 55597637 2017-11-05 21:59:20.967452 mon.ceph02mon mon.1 10.10.1.22:6789/0 362 : audit [DBG] from='client.? 10.10.1.21:0/1655216627' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch
2017-11-05 21:59:21.989364 7f23302e4700 20 mon.ceph03@2(peon).log v55597637 update_from_paxos logging for channel 'audit' to file '/var/log/ceph/ceph.audit.log'
2017-11-05 21:59:21.989369 7f23302e4700  7 mon.ceph03@2(peon).log v55597637 update_from_paxos applying incremental log 55597637 2017-11-05 21:59:20.999972 mon.ceph01mon mon.0 10.10.1.21:6789/0 866 : audit [DBG] from='client.? 10.10.1.21:0/1886445839' entity='client.admin' cmd=[{"prefix": "df", "format": "json"}]: dispatch
2017-11-05 21:59:21.989374 7f23302e4700 20 mon.ceph03@2(peon).log v55597637 update_from_paxos logging for channel 'audit' to file '/var/log/ceph/ceph.audit.log'
2017-11-05 21:59:21.989394 7f23302e4700 15 mon.ceph03@2(peon).log v55597637 update_from_paxos logging for 0 channels
2017-11-05 21:59:21.989400 7f23302e4700 10 mon.ceph03@2(peon).log v55597637 check_subs
2017-11-05 21:59:21.989443 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(monmap 1..14) refresh
2017-11-05 21:59:21.989466 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(auth 26252..26332) refresh
2017-11-05 21:59:21.989473 7f23302e4700 10 mon.ceph03@2(peon).auth v26332 update_from_paxos
2017-11-05 21:59:21.989511 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgr 1..394) refresh
2017-11-05 21:59:21.989563 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgrstat 56609513..56609638) refresh
2017-11-05 21:59:21.989569 7f23302e4700 10 mon.ceph03@2(peon).mgrstat  56609638
2017-11-05 21:59:21.989611 7f23302e4700 10 mon.ceph03@2(peon).mgrstat update_from_paxos v56609638 service_map e1
2017-11-05 21:59:21.989617 7f23302e4700 10 mon.ceph03@2(peon).mgrstat check_subs
2017-11-05 21:59:21.989619 7f23302e4700 20 mon.ceph03@2(peon).mgrstat update_logger
2017-11-05 21:59:21.989649 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(health 1..24) refresh
2017-11-05 21:59:21.989656 7f23302e4700 10 mon.ceph03@2(peon).health update_from_paxos
2017-11-05 21:59:21.989671 7f23302e4700 20 mon.ceph03@2(peon).health dump:{
    "quorum_health": {},
    "leader_health": {}
}

2017-11-05 21:59:21.989686 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(pgmap 56600067..56600783) post_refresh
2017-11-05 21:59:21.989688 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mdsmap 1..1) post_refresh
2017-11-05 21:59:21.989690 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(osdmap 503..1105) post_refresh
2017-11-05 21:59:21.989692 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(logm 55596915..55597637) post_refresh
2017-11-05 21:59:21.989693 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(monmap 1..14) post_refresh
2017-11-05 21:59:21.989695 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(auth 26252..26332) post_refresh
2017-11-05 21:59:21.989696 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgr 1..394) post_refresh
2017-11-05 21:59:21.989700 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgrstat 56609513..56609638) post_refresh
2017-11-05 21:59:21.989701 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(health 1..24) post_refresh
2017-11-05 21:59:21.989790 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
2017-11-05 21:59:21.989799 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
2017-11-05 21:59:21.989803 7f23302e4700 20 is_capable service=mon command= read on cap allow *
2017-11-05 21:59:21.989806 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:21.989808 7f23302e4700 20  allow all
2017-11-05 21:59:21.989810 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
2017-11-05 21:59:21.989811 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:21.989813 7f23302e4700 20  allow all
2017-11-05 21:59:21.989819 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907749) handle_lease on 112907749 now 2017-11-05 21:59:26.947993
2017-11-05 21:59:21.989851 7f23302e4700 20 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907749) reset_lease_timeout - setting timeout event
2017-11-05 21:59:21.989863 7f23302e4700  5 mon.ceph03@2(peon).monmap v14 apply_mon_features wait for service to be writeable
2017-11-05 21:59:21.989866 7f23302e4700  5 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907749) is_readable = 1 - now=2017-11-05 21:59:21.989866 lease_expire=2017-11-05 21:59:26.947993 has v0 lc 112907749
2017-11-05 21:59:21.989898 7f232c2dc700 10 _calc_signature seq 33 front_crc_ = 1097711905 middle_crc = 0 data_crc = 0 sig = 17535115145835410118
2017-11-05 21:59:21.989913 7f232c2dc700 20 Putting signature in client message(seq # 33): sig = 17535115145835410118
2017-11-05 21:59:22.692236 7f2332ae9700 11 mon.ceph03@2(peon) e14 tick
2017-11-05 21:59:22.692259 7f2332ae9700 10 mon.ceph03@2(peon).mds e1
2017-11-05 21:59:22.692264 7f2332ae9700 10 mon.ceph03@2(peon).osd e1105 e1105: 9 total, 9 up, 9 in
2017-11-05 21:59:22.692268 7f2332ae9700 10 mon.ceph03@2(peon).log v55597637 log
2017-11-05 21:59:22.692271 7f2332ae9700 10 mon.ceph03@2(peon).auth v26332 auth
2017-11-05 21:59:22.692275 7f2332ae9700 10 mon.ceph03@2(peon).health tick
2017-11-05 21:59:22.692278 7f2332ae9700 20 mon.ceph03@2(peon).health check_member_health
2017-11-05 21:59:22.692359 7f2332ae9700 10 mon.ceph03@2(peon).health check_member_health avail 92% total 277 GB, used 5676 MB, avail 257 GB
2017-11-05 21:59:22.692371 7f2332ae9700 20 mon.ceph03@2(peon) e14 sync_trim_providers
2017-11-05 21:59:22.898492 7f232d2de700 10 _calc_signature seq 4 front_crc_ = 2516153275 middle_crc = 0 data_crc = 0 sig = 10573281299783805524
2017-11-05 21:59:22.898512 7f232d2de700 20 Putting signature in client message(seq # 4): sig = 10573281299783805524
2017-11-05 21:59:22.990920 7f232c2dc700 10 _calc_signature seq 248515267 front_crc_ = 2763070699 middle_crc = 0 data_crc = 0 sig = 14556592154937786364
2017-11-05 21:59:22.991024 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
2017-11-05 21:59:22.991034 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
2017-11-05 21:59:22.991038 7f23302e4700 20 is_capable service=mon command= read on cap allow *
2017-11-05 21:59:22.991042 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:22.991043 7f23302e4700 20  allow all
2017-11-05 21:59:22.991047 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
2017-11-05 21:59:22.991048 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:22.991049 7f23302e4700 20  allow all
2017-11-05 21:59:22.991057 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907749) handle_begin paxos(begin lc 112907749 fc 0 pn 13700 opn 0) v4
2017-11-05 21:59:22.991061 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos updating c 112907080..112907749) accepting value for 112907750 pn 13700
2017-11-05 21:59:23.029939 7f232c2dc700 10 _calc_signature seq 34 front_crc_ = 2366506105 middle_crc = 0 data_crc = 0 sig = 4827263851779999549
2017-11-05 21:59:23.029956 7f232c2dc700 20 Putting signature in client message(seq # 34): sig = 4827263851779999549
2017-11-05 21:59:23.068457 7f232c2dc700 10 _calc_signature seq 248515268 front_crc_ = 621179324 middle_crc = 0 data_crc = 0 sig = 10355367633632527010
2017-11-05 21:59:23.068555 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
2017-11-05 21:59:23.068564 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
2017-11-05 21:59:23.068573 7f23302e4700 20 is_capable service=mon command= read on cap allow *
2017-11-05 21:59:23.068576 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:23.068577 7f23302e4700 20  allow all
2017-11-05 21:59:23.068579 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
2017-11-05 21:59:23.068580 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:23.068582 7f23302e4700 20  allow all
2017-11-05 21:59:23.068587 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos updating c 112907080..112907749) handle_commit on 112907750
2017-11-05 21:59:23.068593 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos updating c 112907080..112907750) store_state [112907750..112907750]
2017-11-05 21:59:23.068805 7f232c2dc700 10 _calc_signature seq 248515269 front_crc_ = 2140618806 middle_crc = 0 data_crc = 0 sig = 6581431197576721299
2017-11-05 21:59:23.107515 7f23302e4700 10 mon.ceph03@2(peon) e14 refresh_from_paxos
2017-11-05 21:59:23.107551 7f23302e4700 10 mon.ceph03@2(peon) e14 refresh_from_paxos no cluster_fingerprint
2017-11-05 21:59:23.107591 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(pgmap 56600067..56600783) refresh
2017-11-05 21:59:23.107645 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mdsmap 1..1) refresh
2017-11-05 21:59:23.107695 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(osdmap 503..1105) refresh
2017-11-05 21:59:23.107731 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(logm 55596915..55597637) refresh
2017-11-05 21:59:23.107738 7f23302e4700 10 mon.ceph03@2(peon).log v55597637 update_from_paxos
2017-11-05 21:59:23.107741 7f23302e4700 10 mon.ceph03@2(peon).log v55597637 update_from_paxos version 55597637 summary v 55597637
2017-11-05 21:59:23.107783 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(monmap 1..14) refresh
2017-11-05 21:59:23.107810 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(auth 26252..26332) refresh
2017-11-05 21:59:23.107817 7f23302e4700 10 mon.ceph03@2(peon).auth v26332 update_from_paxos
2017-11-05 21:59:23.107860 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgr 1..394) refresh
2017-11-05 21:59:23.107922 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgrstat 56609513..56609639) refresh
2017-11-05 21:59:23.107930 7f23302e4700 10 mon.ceph03@2(peon).mgrstat  56609639
2017-11-05 21:59:23.107981 7f23302e4700 10 mon.ceph03@2(peon).mgrstat update_from_paxos v56609639 service_map e1
2017-11-05 21:59:23.107987 7f23302e4700 10 mon.ceph03@2(peon).mgrstat check_subs
2017-11-05 21:59:23.107989 7f23302e4700 20 mon.ceph03@2(peon).mgrstat update_logger
2017-11-05 21:59:23.108023 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(health 1..24) refresh
2017-11-05 21:59:23.108031 7f23302e4700 10 mon.ceph03@2(peon).health update_from_paxos
2017-11-05 21:59:23.108048 7f23302e4700 20 mon.ceph03@2(peon).health dump:{
    "quorum_health": {},
    "leader_health": {}
}

2017-11-05 21:59:23.108064 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(pgmap 56600067..56600783) post_refresh
2017-11-05 21:59:23.108067 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mdsmap 1..1) post_refresh
2017-11-05 21:59:23.108068 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(osdmap 503..1105) post_refresh
2017-11-05 21:59:23.108070 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(logm 55596915..55597637) post_refresh
2017-11-05 21:59:23.108072 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(monmap 1..14) post_refresh
2017-11-05 21:59:23.108074 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(auth 26252..26332) post_refresh
2017-11-05 21:59:23.108075 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgr 1..394) post_refresh
2017-11-05 21:59:23.108092 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgrstat 56609513..56609639) post_refresh
2017-11-05 21:59:23.108094 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(health 1..24) post_refresh
2017-11-05 21:59:23.108223 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
2017-11-05 21:59:23.108232 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
2017-11-05 21:59:23.108247 7f23302e4700 20 is_capable service=mon command= read on cap allow *
2017-11-05 21:59:23.108253 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:23.108255 7f23302e4700 20  allow all
2017-11-05 21:59:23.108257 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
2017-11-05 21:59:23.108259 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:23.108260 7f23302e4700 20  allow all
2017-11-05 21:59:23.108265 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907750) handle_lease on 112907750 now 2017-11-05 21:59:28.066966
2017-11-05 21:59:23.108297 7f23302e4700 20 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907750) reset_lease_timeout - setting timeout event
2017-11-05 21:59:23.108308 7f23302e4700  5 mon.ceph03@2(peon).monmap v14 apply_mon_features wait for service to be writeable
2017-11-05 21:59:23.108312 7f23302e4700  5 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907750) is_readable = 1 - now=2017-11-05 21:59:23.108313 lease_expire=2017-11-05 21:59:28.066966 has v0 lc 112907750
2017-11-05 21:59:23.108370 7f232c2dc700 10 _calc_signature seq 35 front_crc_ = 2829686340 middle_crc = 0 data_crc = 0 sig = 14457767592524038708
2017-11-05 21:59:23.108388 7f232c2dc700 20 Putting signature in client message(seq # 35): sig = 14457767592524038708
2017-11-05 21:59:23.247750 7f232cadd700 10 _calc_signature seq 123007633 front_crc_ = 992124289 middle_crc = 0 data_crc = 0 sig = 9589994617269070540
2017-11-05 21:59:23.247874 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930c00 for mon.1 10.10.1.22:6789/0
2017-11-05 21:59:23.247884 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
2017-11-05 21:59:23.247887 7f23302e4700 20 is_capable service=mon command= read on cap allow *
2017-11-05 21:59:23.247890 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:23.247891 7f23302e4700 20  allow all
2017-11-05 21:59:23.247895 7f23302e4700 10 mon.ceph03@2(peon).data_health(344) service_dispatch_op mon_health( service 1 op tell e 0 r 0 ) v1
2017-11-05 21:59:23.247900 7f23302e4700 10 mon.ceph03@2(peon).data_health(344) handle_tell mon_health( service 1 op tell e 0 r 0 ) v1
2017-11-05 21:59:24.275431 7f232c2dc700 10 _calc_signature seq 248515270 front_crc_ = 1916288138 middle_crc = 0 data_crc = 0 sig = 5259317027298220120
2017-11-05 21:59:24.275563 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
2017-11-05 21:59:24.275574 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
2017-11-05 21:59:24.275578 7f23302e4700 20 is_capable service=mon command= read on cap allow *
2017-11-05 21:59:24.275583 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:24.275585 7f23302e4700 20  allow all
2017-11-05 21:59:24.275587 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
2017-11-05 21:59:24.275589 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:24.275590 7f23302e4700 20  allow all
2017-11-05 21:59:24.275596 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907750) handle_begin paxos(begin lc 112907750 fc 0 pn 13700 opn 0) v4
2017-11-05 21:59:24.275602 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos updating c 112907080..112907750) accepting value for 112907751 pn 13700
2017-11-05 21:59:24.314433 7f232c2dc700 10 _calc_signature seq 36 front_crc_ = 1221148835 middle_crc = 0 data_crc = 0 sig = 13457619710782546457
2017-11-05 21:59:24.314453 7f232c2dc700 20 Putting signature in client message(seq # 36): sig = 13457619710782546457
2017-11-05 21:59:24.353252 7f232c2dc700 10 _calc_signature seq 248515271 front_crc_ = 4266369837 middle_crc = 0 data_crc = 0 sig = 17507013993340613918
2017-11-05 21:59:24.353368 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
2017-11-05 21:59:24.353377 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
2017-11-05 21:59:24.353380 7f23302e4700 20 is_capable service=mon command= read on cap allow *
2017-11-05 21:59:24.353383 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:24.353386 7f23302e4700 20  allow all
2017-11-05 21:59:24.353388 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
2017-11-05 21:59:24.353390 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:24.353391 7f23302e4700 20  allow all
2017-11-05 21:59:24.353396 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos updating c 112907080..112907750) handle_commit on 112907751
2017-11-05 21:59:24.353401 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos updating c 112907080..112907751) store_state [112907751..112907751]
2017-11-05 21:59:24.353583 7f232c2dc700 10 _calc_signature seq 248515272 front_crc_ = 1064734685 middle_crc = 0 data_crc = 0 sig = 10177169538778707476
2017-11-05 21:59:24.391989 7f23302e4700 10 mon.ceph03@2(peon) e14 refresh_from_paxos
2017-11-05 21:59:24.392025 7f23302e4700 10 mon.ceph03@2(peon) e14 refresh_from_paxos no cluster_fingerprint
2017-11-05 21:59:24.392060 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(pgmap 56600067..56600783) refresh
2017-11-05 21:59:24.392138 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mdsmap 1..1) refresh
2017-11-05 21:59:24.392193 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(osdmap 503..1105) refresh
2017-11-05 21:59:24.392229 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(logm 55596915..55597637) refresh
2017-11-05 21:59:24.392236 7f23302e4700 10 mon.ceph03@2(peon).log v55597637 update_from_paxos
2017-11-05 21:59:24.392239 7f23302e4700 10 mon.ceph03@2(peon).log v55597637 update_from_paxos version 55597637 summary v 55597637
2017-11-05 21:59:24.392282 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(monmap 1..14) refresh
2017-11-05 21:59:24.392309 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(auth 26252..26332) refresh
2017-11-05 21:59:24.392315 7f23302e4700 10 mon.ceph03@2(peon).auth v26332 update_from_paxos
2017-11-05 21:59:24.392359 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgr 1..394) refresh
2017-11-05 21:59:24.392420 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgrstat 56609513..56609640) refresh
2017-11-05 21:59:24.392427 7f23302e4700 10 mon.ceph03@2(peon).mgrstat  56609640
2017-11-05 21:59:24.392479 7f23302e4700 10 mon.ceph03@2(peon).mgrstat update_from_paxos v56609640 service_map e1
2017-11-05 21:59:24.392485 7f23302e4700 10 mon.ceph03@2(peon).mgrstat check_subs
2017-11-05 21:59:24.392488 7f23302e4700 20 mon.ceph03@2(peon).mgrstat update_logger
2017-11-05 21:59:24.392523 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(health 1..24) refresh
2017-11-05 21:59:24.392530 7f23302e4700 10 mon.ceph03@2(peon).health update_from_paxos
2017-11-05 21:59:24.392547 7f23302e4700 20 mon.ceph03@2(peon).health dump:{
    "quorum_health": {},
    "leader_health": {}
}

2017-11-05 21:59:24.392563 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(pgmap 56600067..56600783) post_refresh
2017-11-05 21:59:24.392565 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mdsmap 1..1) post_refresh
2017-11-05 21:59:24.392567 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(osdmap 503..1105) post_refresh
2017-11-05 21:59:24.392569 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(logm 55596915..55597637) post_refresh
2017-11-05 21:59:24.392571 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(monmap 1..14) post_refresh
2017-11-05 21:59:24.392573 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(auth 26252..26332) post_refresh
2017-11-05 21:59:24.392575 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgr 1..394) post_refresh
2017-11-05 21:59:24.392576 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgrstat 56609513..56609640) post_refresh
2017-11-05 21:59:24.392578 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(health 1..24) post_refresh
2017-11-05 21:59:24.392682 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
2017-11-05 21:59:24.392690 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
2017-11-05 21:59:24.392693 7f23302e4700 20 is_capable service=mon command= read on cap allow *
2017-11-05 21:59:24.392696 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:24.392697 7f23302e4700 20  allow all
2017-11-05 21:59:24.392698 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
2017-11-05 21:59:24.392716 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:24.392718 7f23302e4700 20  allow all
2017-11-05 21:59:24.392722 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907751) handle_lease on 112907751 now 2017-11-05 21:59:29.351713
2017-11-05 21:59:24.392755 7f23302e4700 20 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907751) reset_lease_timeout - setting timeout event
2017-11-05 21:59:24.392765 7f23302e4700  5 mon.ceph03@2(peon).monmap v14 apply_mon_features wait for service to be writeable
2017-11-05 21:59:24.392769 7f23302e4700  5 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907751) is_readable = 1 - now=2017-11-05 21:59:24.392769 lease_expire=2017-11-05 21:59:29.351713 has v0 lc 112907751
2017-11-05 21:59:24.392824 7f232c2dc700 10 _calc_signature seq 37 front_crc_ = 2948331090 middle_crc = 0 data_crc = 0 sig = 11223045120982073528
2017-11-05 21:59:24.392842 7f232c2dc700 20 Putting signature in client message(seq # 37): sig = 11223045120982073528
2017-11-05 21:59:26.268592 7f232c2dc700 10 _calc_signature seq 248515273 front_crc_ = 1773435773 middle_crc = 0 data_crc = 0 sig = 5758794856142174140
2017-11-05 21:59:26.268737 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
2017-11-05 21:59:26.268748 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
2017-11-05 21:59:26.268752 7f23302e4700 20 is_capable service=mon command= read on cap allow *
2017-11-05 21:59:26.268755 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:26.268757 7f23302e4700 20  allow all
2017-11-05 21:59:26.268759 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
2017-11-05 21:59:26.268760 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:26.268761 7f23302e4700 20  allow all
2017-11-05 21:59:26.268768 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907751) handle_begin paxos(begin lc 112907751 fc 0 pn 13700 opn 0) v4
2017-11-05 21:59:26.268773 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos updating c 112907080..112907751) accepting value for 112907752 pn 13700
2017-11-05 21:59:26.307408 7f232c2dc700 10 _calc_signature seq 38 front_crc_ = 382545443 middle_crc = 0 data_crc = 0 sig = 6576343383518060326
2017-11-05 21:59:26.307427 7f232c2dc700 20 Putting signature in client message(seq # 38): sig = 6576343383518060326
2017-11-05 21:59:26.346943 7f232c2dc700 10 _calc_signature seq 248515274 front_crc_ = 3233349729 middle_crc = 0 data_crc = 0 sig = 4287989692588765716
2017-11-05 21:59:26.347015 7f232c2dc700 10 _calc_signature seq 248515275 front_crc_ = 3164033032 middle_crc = 0 data_crc = 0 sig = 4655778565917331996
2017-11-05 21:59:26.347056 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
2017-11-05 21:59:26.347065 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
2017-11-05 21:59:26.347068 7f23302e4700 20 is_capable service=mon command= read on cap allow *
2017-11-05 21:59:26.347071 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:26.347072 7f23302e4700 20  allow all
2017-11-05 21:59:26.347074 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
2017-11-05 21:59:26.347075 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:26.347076 7f23302e4700 20  allow all
2017-11-05 21:59:26.347083 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos updating c 112907080..112907751) handle_commit on 112907752
2017-11-05 21:59:26.347088 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos updating c 112907080..112907752) store_state [112907752..112907752]
2017-11-05 21:59:26.385029 7f23302e4700 10 mon.ceph03@2(peon) e14 refresh_from_paxos
2017-11-05 21:59:26.385064 7f23302e4700 10 mon.ceph03@2(peon) e14 refresh_from_paxos no cluster_fingerprint
2017-11-05 21:59:26.385095 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(pgmap 56600067..56600783) refresh
2017-11-05 21:59:26.385143 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mdsmap 1..1) refresh
2017-11-05 21:59:26.385189 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(osdmap 503..1105) refresh
2017-11-05 21:59:26.385223 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(logm 55596915..55597637) refresh
2017-11-05 21:59:26.385230 7f23302e4700 10 mon.ceph03@2(peon).log v55597637 update_from_paxos
2017-11-05 21:59:26.385233 7f23302e4700 10 mon.ceph03@2(peon).log v55597637 update_from_paxos version 55597637 summary v 55597637
2017-11-05 21:59:26.385275 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(monmap 1..14) refresh
2017-11-05 21:59:26.385301 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(auth 26252..26332) refresh
2017-11-05 21:59:26.385307 7f23302e4700 10 mon.ceph03@2(peon).auth v26332 update_from_paxos
2017-11-05 21:59:26.385348 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgr 1..394) refresh
2017-11-05 21:59:26.385405 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgrstat 56609513..56609641) refresh
2017-11-05 21:59:26.385412 7f23302e4700 10 mon.ceph03@2(peon).mgrstat  56609641
2017-11-05 21:59:26.385461 7f23302e4700 10 mon.ceph03@2(peon).mgrstat update_from_paxos v56609641 service_map e1
2017-11-05 21:59:26.385467 7f23302e4700 10 mon.ceph03@2(peon).mgrstat check_subs
2017-11-05 21:59:26.385469 7f23302e4700 20 mon.ceph03@2(peon).mgrstat update_logger
2017-11-05 21:59:26.385501 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(health 1..24) refresh
2017-11-05 21:59:26.385508 7f23302e4700 10 mon.ceph03@2(peon).health update_from_paxos
2017-11-05 21:59:26.385524 7f23302e4700 20 mon.ceph03@2(peon).health dump:{
    "quorum_health": {},
    "leader_health": {}
}

2017-11-05 21:59:26.385541 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(pgmap 56600067..56600783) post_refresh
2017-11-05 21:59:26.385543 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mdsmap 1..1) post_refresh
2017-11-05 21:59:26.385545 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(osdmap 503..1105) post_refresh
2017-11-05 21:59:26.385546 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(logm 55596915..55597637) post_refresh
2017-11-05 21:59:26.385548 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(monmap 1..14) post_refresh
2017-11-05 21:59:26.385549 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(auth 26252..26332) post_refresh
2017-11-05 21:59:26.385551 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgr 1..394) post_refresh
2017-11-05 21:59:26.385552 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgrstat 56609513..56609641) post_refresh
2017-11-05 21:59:26.385554 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(health 1..24) post_refresh
2017-11-05 21:59:26.385642 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
2017-11-05 21:59:26.385650 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
2017-11-05 21:59:26.385653 7f23302e4700 20 is_capable service=mon command= read on cap allow *
2017-11-05 21:59:26.385656 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:26.385657 7f23302e4700 20  allow all
2017-11-05 21:59:26.385659 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
2017-11-05 21:59:26.385660 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:26.385661 7f23302e4700 20  allow all
2017-11-05 21:59:26.385665 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907752) handle_lease on 112907752 now 2017-11-05 21:59:31.344866
2017-11-05 21:59:26.385696 7f23302e4700 20 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907752) reset_lease_timeout - setting timeout event
2017-11-05 21:59:26.385703 7f23302e4700  5 mon.ceph03@2(peon).monmap v14 apply_mon_features wait for service to be writeable
2017-11-05 21:59:26.385706 7f23302e4700  5 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907752) is_readable = 1 - now=2017-11-05 21:59:26.385707 lease_expire=2017-11-05 21:59:31.344866 has v0 lc 112907752
2017-11-05 21:59:26.385739 7f232c2dc700 10 _calc_signature seq 39 front_crc_ = 1024609643 middle_crc = 0 data_crc = 0 sig = 15304902292167068210
2017-11-05 21:59:26.385750 7f232c2dc700 20 Putting signature in client message(seq # 39): sig = 15304902292167068210
2017-11-05 21:59:27.692448 7f2332ae9700 11 mon.ceph03@2(peon) e14 tick
2017-11-05 21:59:27.692469 7f2332ae9700 10 mon.ceph03@2(peon).mds e1
2017-11-05 21:59:27.692474 7f2332ae9700 10 mon.ceph03@2(peon).osd e1105 e1105: 9 total, 9 up, 9 in
2017-11-05 21:59:27.692479 7f2332ae9700 10 mon.ceph03@2(peon).log v55597637 log
2017-11-05 21:59:27.692509 7f2332ae9700 10 mon.ceph03@2(peon).auth v26332 auth
2017-11-05 21:59:27.692513 7f2332ae9700 10 mon.ceph03@2(peon).health tick
2017-11-05 21:59:27.692515 7f2332ae9700 20 mon.ceph03@2(peon).health check_member_health
2017-11-05 21:59:27.692599 7f2332ae9700 10 mon.ceph03@2(peon).health check_member_health avail 92% total 277 GB, used 5676 MB, avail 257 GB
2017-11-05 21:59:27.692609 7f2332ae9700 20 mon.ceph03@2(peon) e14 sync_trim_providers
2017-11-05 21:59:27.898811 7f232d2de700 10 _calc_signature seq 5 front_crc_ = 1497291623 middle_crc = 0 data_crc = 0 sig = 7653944384080933906
2017-11-05 21:59:27.898830 7f232d2de700 20 Putting signature in client message(seq # 5): sig = 7653944384080933906
2017-11-05 21:59:28.273089 7f232c2dc700 10 _calc_signature seq 248515276 front_crc_ = 1793616534 middle_crc = 0 data_crc = 0 sig = 17139277050217223183
2017-11-05 21:59:28.273202 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
2017-11-05 21:59:28.273213 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
2017-11-05 21:59:28.273217 7f23302e4700 20 is_capable service=mon command= read on cap allow *
2017-11-05 21:59:28.273221 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:28.273222 7f23302e4700 20  allow all
2017-11-05 21:59:28.273225 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
2017-11-05 21:59:28.273226 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:28.273227 7f23302e4700 20  allow all
2017-11-05 21:59:28.273235 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907752) handle_begin paxos(begin lc 112907752 fc 0 pn 13700 opn 0) v4
2017-11-05 21:59:28.273240 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos updating c 112907080..112907752) accepting value for 112907753 pn 13700
2017-11-05 21:59:28.311436 7f232c2dc700 10 _calc_signature seq 40 front_crc_ = 2022008981 middle_crc = 0 data_crc = 0 sig = 7818860563918508765
2017-11-05 21:59:28.311453 7f232c2dc700 20 Putting signature in client message(seq # 40): sig = 7818860563918508765
2017-11-05 21:59:28.339873 7f232c2dc700 10 _calc_signature seq 248515277 front_crc_ = 1315244736 middle_crc = 0 data_crc = 0 sig = 7080838384747451889
2017-11-05 21:59:28.339990 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
2017-11-05 21:59:28.340000 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
2017-11-05 21:59:28.340003 7f23302e4700 20 is_capable service=mon command= read on cap allow *
2017-11-05 21:59:28.340006 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:28.340007 7f23302e4700 20  allow all
2017-11-05 21:59:28.340009 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
2017-11-05 21:59:28.340011 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:28.340013 7f23302e4700 20  allow all
2017-11-05 21:59:28.340038 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos updating c 112907080..112907752) handle_commit on 112907753
2017-11-05 21:59:28.340044 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos updating c 112907080..112907753) store_state [112907753..112907753]
2017-11-05 21:59:28.340240 7f232c2dc700 10 _calc_signature seq 248515278 front_crc_ = 7690493 middle_crc = 0 data_crc = 0 sig = 12630673617109963947
2017-11-05 21:59:28.378021 7f23302e4700 10 mon.ceph03@2(peon) e14 refresh_from_paxos
2017-11-05 21:59:28.378058 7f23302e4700 10 mon.ceph03@2(peon) e14 refresh_from_paxos no cluster_fingerprint
2017-11-05 21:59:28.378091 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(pgmap 56600067..56600783) refresh
2017-11-05 21:59:28.378141 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mdsmap 1..1) refresh
2017-11-05 21:59:28.378191 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(osdmap 503..1105) refresh
2017-11-05 21:59:28.378227 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(logm 55596915..55597637) refresh
2017-11-05 21:59:28.378234 7f23302e4700 10 mon.ceph03@2(peon).log v55597637 update_from_paxos
2017-11-05 21:59:28.378238 7f23302e4700 10 mon.ceph03@2(peon).log v55597637 update_from_paxos version 55597637 summary v 55597637
2017-11-05 21:59:28.378280 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(monmap 1..14) refresh
2017-11-05 21:59:28.378308 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(auth 26252..26332) refresh
2017-11-05 21:59:28.378315 7f23302e4700 10 mon.ceph03@2(peon).auth v26332 update_from_paxos
2017-11-05 21:59:28.378359 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgr 1..394) refresh
2017-11-05 21:59:28.378414 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgrstat 56609513..56609642) refresh
2017-11-05 21:59:28.378421 7f23302e4700 10 mon.ceph03@2(peon).mgrstat  56609642
2017-11-05 21:59:28.378472 7f23302e4700 10 mon.ceph03@2(peon).mgrstat update_from_paxos v56609642 service_map e1
2017-11-05 21:59:28.378479 7f23302e4700 10 mon.ceph03@2(peon).mgrstat check_subs
2017-11-05 21:59:28.378481 7f23302e4700 20 mon.ceph03@2(peon).mgrstat update_logger
2017-11-05 21:59:28.378514 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(health 1..24) refresh
2017-11-05 21:59:28.378521 7f23302e4700 10 mon.ceph03@2(peon).health update_from_paxos
2017-11-05 21:59:28.378539 7f23302e4700 20 mon.ceph03@2(peon).health dump:{
    "quorum_health": {},
    "leader_health": {}
}

2017-11-05 21:59:28.378555 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(pgmap 56600067..56600783) post_refresh
2017-11-05 21:59:28.378557 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mdsmap 1..1) post_refresh
2017-11-05 21:59:28.378559 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(osdmap 503..1105) post_refresh
2017-11-05 21:59:28.378561 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(logm 55596915..55597637) post_refresh
2017-11-05 21:59:28.378562 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(monmap 1..14) post_refresh
2017-11-05 21:59:28.378564 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(auth 26252..26332) post_refresh
2017-11-05 21:59:28.378566 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgr 1..394) post_refresh
2017-11-05 21:59:28.378567 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgrstat 56609513..56609642) post_refresh
2017-11-05 21:59:28.378570 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(health 1..24) post_refresh
2017-11-05 21:59:28.378661 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
2017-11-05 21:59:28.378666 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
2017-11-05 21:59:28.378669 7f23302e4700 20 is_capable service=mon command= read on cap allow *
2017-11-05 21:59:28.378674 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:28.378675 7f23302e4700 20  allow all
2017-11-05 21:59:28.378677 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
2017-11-05 21:59:28.378678 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:28.378679 7f23302e4700 20  allow all
2017-11-05 21:59:28.378684 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907753) handle_lease on 112907753 now 2017-11-05 21:59:33.338364
2017-11-05 21:59:28.378716 7f23302e4700 20 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907753) reset_lease_timeout - setting timeout event
2017-11-05 21:59:28.378723 7f23302e4700  5 mon.ceph03@2(peon).monmap v14 apply_mon_features wait for service to be writeable
2017-11-05 21:59:28.378726 7f23302e4700  5 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907753) is_readable = 1 - now=2017-11-05 21:59:28.378727 lease_expire=2017-11-05 21:59:33.338364 has v0 lc 112907753
2017-11-05 21:59:28.378784 7f232c2dc700 10 _calc_signature seq 41 front_crc_ = 4223284809 middle_crc = 0 data_crc = 0 sig = 2790373739108272567
2017-11-05 21:59:28.378801 7f232c2dc700 20 Putting signature in client message(seq # 41): sig = 2790373739108272567
2017-11-05 21:59:30.266286 7f232c2dc700 10 _calc_signature seq 248515279 front_crc_ = 4264070926 middle_crc = 0 data_crc = 0 sig = 17570471818085489431
2017-11-05 21:59:30.266414 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
2017-11-05 21:59:30.266424 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
2017-11-05 21:59:30.266429 7f23302e4700 20 is_capable service=mon command= read on cap allow *
2017-11-05 21:59:30.266433 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:30.266434 7f23302e4700 20  allow all
2017-11-05 21:59:30.266436 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
2017-11-05 21:59:30.266438 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:30.266439 7f23302e4700 20  allow all
2017-11-05 21:59:30.266447 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907753) handle_begin paxos(begin lc 112907753 fc 0 pn 13700 opn 0) v4
2017-11-05 21:59:30.266453 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos updating c 112907080..112907753) accepting value for 112907754 pn 13700
2017-11-05 21:59:30.304357 7f232c2dc700 10 _calc_signature seq 42 front_crc_ = 3454116069 middle_crc = 0 data_crc = 0 sig = 17816842680307483109
2017-11-05 21:59:30.304370 7f232c2dc700 20 Putting signature in client message(seq # 42): sig = 17816842680307483109
2017-11-05 21:59:30.344058 7f232c2dc700 10 _calc_signature seq 248515280 front_crc_ = 2165871158 middle_crc = 0 data_crc = 0 sig = 17177650932287336935
2017-11-05 21:59:30.344156 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
2017-11-05 21:59:30.344168 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
2017-11-05 21:59:30.344171 7f23302e4700 20 is_capable service=mon command= read on cap allow *
2017-11-05 21:59:30.344174 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:30.344195 7f23302e4700 20  allow all
2017-11-05 21:59:30.344197 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
2017-11-05 21:59:30.344199 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:30.344200 7f23302e4700 20  allow all
2017-11-05 21:59:30.344211 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos updating c 112907080..112907753) handle_commit on 112907754
2017-11-05 21:59:30.344217 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos updating c 112907080..112907754) store_state [112907754..112907754]
2017-11-05 21:59:30.344428 7f232c2dc700 10 _calc_signature seq 248515281 front_crc_ = 1698768348 middle_crc = 0 data_crc = 0 sig = 16000968398340812496
2017-11-05 21:59:30.382036 7f23302e4700 10 mon.ceph03@2(peon) e14 refresh_from_paxos
2017-11-05 21:59:30.382072 7f23302e4700 10 mon.ceph03@2(peon) e14 refresh_from_paxos no cluster_fingerprint
2017-11-05 21:59:30.382106 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(pgmap 56600067..56600783) refresh
2017-11-05 21:59:30.382153 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mdsmap 1..1) refresh
2017-11-05 21:59:30.382204 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(osdmap 503..1105) refresh
2017-11-05 21:59:30.382251 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(logm 55596915..55597637) refresh
2017-11-05 21:59:30.382258 7f23302e4700 10 mon.ceph03@2(peon).log v55597637 update_from_paxos
2017-11-05 21:59:30.382261 7f23302e4700 10 mon.ceph03@2(peon).log v55597637 update_from_paxos version 55597637 summary v 55597637
2017-11-05 21:59:30.382301 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(monmap 1..14) refresh
2017-11-05 21:59:30.382326 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(auth 26252..26332) refresh
2017-11-05 21:59:30.382333 7f23302e4700 10 mon.ceph03@2(peon).auth v26332 update_from_paxos
2017-11-05 21:59:30.382374 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgr 1..394) refresh
2017-11-05 21:59:30.382426 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgrstat 56609513..56609643) refresh
2017-11-05 21:59:30.382434 7f23302e4700 10 mon.ceph03@2(peon).mgrstat  56609643
2017-11-05 21:59:30.382491 7f23302e4700 10 mon.ceph03@2(peon).mgrstat update_from_paxos v56609643 service_map e1
2017-11-05 21:59:30.382497 7f23302e4700 10 mon.ceph03@2(peon).mgrstat check_subs
2017-11-05 21:59:30.382502 7f23302e4700 20 mon.ceph03@2(peon).mgrstat update_logger
2017-11-05 21:59:30.382534 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(health 1..24) refresh
2017-11-05 21:59:30.382541 7f23302e4700 10 mon.ceph03@2(peon).health update_from_paxos
2017-11-05 21:59:30.382558 7f23302e4700 20 mon.ceph03@2(peon).health dump:{
    "quorum_health": {},
    "leader_health": {}
}

2017-11-05 21:59:30.382573 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(pgmap 56600067..56600783) post_refresh
2017-11-05 21:59:30.382575 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mdsmap 1..1) post_refresh
2017-11-05 21:59:30.382577 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(osdmap 503..1105) post_refresh
2017-11-05 21:59:30.382579 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(logm 55596915..55597637) post_refresh
2017-11-05 21:59:30.382580 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(monmap 1..14) post_refresh
2017-11-05 21:59:30.382582 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(auth 26252..26332) post_refresh
2017-11-05 21:59:30.382583 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgr 1..394) post_refresh
2017-11-05 21:59:30.382585 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgrstat 56609513..56609643) post_refresh
2017-11-05 21:59:30.382587 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(health 1..24) post_refresh
2017-11-05 21:59:30.382678 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
2017-11-05 21:59:30.382685 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
2017-11-05 21:59:30.382688 7f23302e4700 20 is_capable service=mon command= read on cap allow *
2017-11-05 21:59:30.382690 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:30.382691 7f23302e4700 20  allow all
2017-11-05 21:59:30.382693 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
2017-11-05 21:59:30.382694 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:30.382696 7f23302e4700 20  allow all
2017-11-05 21:59:30.382700 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907754) handle_lease on 112907754 now 2017-11-05 21:59:35.342590
2017-11-05 21:59:30.382733 7f23302e4700 20 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907754) reset_lease_timeout - setting timeout event
2017-11-05 21:59:30.382742 7f23302e4700  5 mon.ceph03@2(peon).monmap v14 apply_mon_features wait for service to be writeable
2017-11-05 21:59:30.382745 7f23302e4700  5 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907754) is_readable = 1 - now=2017-11-05 21:59:30.382746 lease_expire=2017-11-05 21:59:35.342590 has v0 lc 112907754
2017-11-05 21:59:30.382781 7f232c2dc700 10 _calc_signature seq 43 front_crc_ = 2373860913 middle_crc = 0 data_crc = 0 sig = 14432907233122927278
2017-11-05 21:59:30.382797 7f232c2dc700 20 Putting signature in client message(seq # 43): sig = 14432907233122927278
2017-11-05 21:59:30.724292 7f232d2de700 10 mon.ceph03@2(peon) e14 ms_verify_authorizer 10.10.1.21:0/1168097192 client protocol 0
2017-11-05 21:59:30.724316 7f232d2de700 10 In get_auth_session_handler for protocol 0
2017-11-05 21:59:30.724638 7f23302e4700 10 mon.ceph03@2(peon) e14 _ms_dispatch new session 0x7f2342933600 MonSession(client.? 10.10.1.21:0/1168097192 is open ) features 0x1ffddff8eea4fffb
2017-11-05 21:59:30.724650 7f23302e4700 20 mon.ceph03@2(peon) e14  caps
2017-11-05 21:59:30.724653 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(auth 26252..26332) dispatch 0x7f2342f14000 auth(proto 0 30 bytes epoch 0) v1 from client.? 10.10.1.21:0/1168097192 con 0x7f2342f09000
2017-11-05 21:59:30.724661 7f23302e4700  5 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907754) is_readable = 1 - now=2017-11-05 21:59:30.724661 lease_expire=2017-11-05 21:59:35.342590 has v0 lc 112907754
2017-11-05 21:59:30.724671 7f23302e4700 10 mon.ceph03@2(peon).auth v26332 preprocess_query auth(proto 0 30 bytes epoch 0) v1 from client.? 10.10.1.21:0/1168097192
2017-11-05 21:59:30.724676 7f23302e4700 10 mon.ceph03@2(peon).auth v26332 prep_auth() blob_size=30
2017-11-05 21:59:30.724691 7f23302e4700 10 mon.ceph03@2(peon).auth v26332 AuthMonitor::assign_global_id m=auth(proto 0 30 bytes epoch 0) v1 mon=2/3 last_allocated=55234096 max_global_id=55244096
2017-11-05 21:59:30.724695 7f23302e4700 10 mon.ceph03@2(peon).auth v26332 next_global_id should be 55234100
2017-11-05 21:59:30.724755 7f23302e4700 10 cephx server client.admin: start_session server_challenge d3423fceec82c2fe
2017-11-05 21:59:30.724764 7f23302e4700  2 mon.ceph03@2(peon) e14 send_reply 0x7f2342e9b8c0 0x7f234296d900 auth_reply(proto 2 0 (0) Success) v1
2017-11-05 21:59:30.725179 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342933600 for client.? 10.10.1.21:0/1168097192
2017-11-05 21:59:30.725185 7f23302e4700 20 mon.ceph03@2(peon) e14  caps
2017-11-05 21:59:30.725188 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(auth 26252..26332) dispatch 0x7f234296d900 auth(proto 2 32 bytes epoch 0) v1 from client.? 10.10.1.21:0/1168097192 con 0x7f2342f09000
2017-11-05 21:59:30.725195 7f23302e4700  5 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907754) is_readable = 1 - now=2017-11-05 21:59:30.725195 lease_expire=2017-11-05 21:59:35.342590 has v0 lc 112907754
2017-11-05 21:59:30.725204 7f23302e4700 10 mon.ceph03@2(peon).auth v26332 preprocess_query auth(proto 2 32 bytes epoch 0) v1 from client.? 10.10.1.21:0/1168097192
2017-11-05 21:59:30.725210 7f23302e4700 10 mon.ceph03@2(peon).auth v26332 prep_auth() blob_size=32
2017-11-05 21:59:30.725213 7f23302e4700 10 cephx server client.admin: handle_request get_auth_session_key for client.admin
2017-11-05 21:59:30.725247 7f23302e4700 20 cephx server client.admin:  checking key: req.key=e3c606c55c103797 expected_key=e3c606c55c103797
2017-11-05 21:59:30.725324 7f23302e4700 10 cephx: build_service_ticket_reply encoding 1 tickets with secret AQAh1Z5VjAN3ARAAw+R6HvdEfcFucLX1YSmG4Q==
2017-11-05 21:59:30.725352 7f23302e4700 10 cephx: build_service_ticket service auth secret_id 1699 ticket_info.ticket.name=client.admin
2017-11-05 21:59:30.725372 7f23302e4700 10 cephx keyserverdata: get_caps: name=client.admin
2017-11-05 21:59:30.725378 7f23302e4700 10 cephx keyserverdata: get_secret: num of caps=4
2017-11-05 21:59:30.725432 7f23302e4700  2 mon.ceph03@2(peon) e14 send_reply 0x7f2342e9b8c0 0x7f2342f14000 auth_reply(proto 2 0 (0) Success) v1
2017-11-05 21:59:30.725481 7f23302e4700 10 mon.ceph03@2(peon) e14 ms_handle_reset 0x7f2342f09000 10.10.1.21:0/1168097192
2017-11-05 21:59:30.725489 7f23302e4700 10 mon.ceph03@2(peon) e14 reset/close on session client.? 10.10.1.21:0/1168097192
2017-11-05 21:59:30.725494 7f23302e4700 10 mon.ceph03@2(peon) e14 remove_session 0x7f2342933600 client.? 10.10.1.21:0/1168097192 features 0x1ffddff8eea4fffb
2017-11-05 21:59:30.772416 7f232d2de700 10 mon.ceph03@2(peon) e14 ms_verify_authorizer 10.10.1.21:0/1317201829 client protocol 0
2017-11-05 21:59:30.772438 7f232d2de700 10 In get_auth_session_handler for protocol 0
2017-11-05 21:59:30.772666 7f23302e4700 10 mon.ceph03@2(peon) e14 _ms_dispatch new session 0x7f2342933600 MonSession(client.? 10.10.1.21:0/1317201829 is open ) features 0x1ffddff8eea4fffb
2017-11-05 21:59:30.772677 7f23302e4700 20 mon.ceph03@2(peon) e14  caps
2017-11-05 21:59:30.772680 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(auth 26252..26332) dispatch 0x7f2342f14500 auth(proto 0 30 bytes epoch 0) v1 from client.? 10.10.1.21:0/1317201829 con 0x7f2342f07800
2017-11-05 21:59:30.772686 7f23302e4700  5 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907754) is_readable = 1 - now=2017-11-05 21:59:30.772687 lease_expire=2017-11-05 21:59:35.342590 has v0 lc 112907754
2017-11-05 21:59:30.772695 7f23302e4700 10 mon.ceph03@2(peon).auth v26332 preprocess_query auth(proto 0 30 bytes epoch 0) v1 from client.? 10.10.1.21:0/1317201829
2017-11-05 21:59:30.772700 7f23302e4700 10 mon.ceph03@2(peon).auth v26332 prep_auth() blob_size=30
2017-11-05 21:59:30.772709 7f23302e4700 10 mon.ceph03@2(peon).auth v26332 AuthMonitor::assign_global_id m=auth(proto 0 30 bytes epoch 0) v1 mon=2/3 last_allocated=55234100 max_global_id=55244096
2017-11-05 21:59:30.772714 7f23302e4700 10 mon.ceph03@2(peon).auth v26332 next_global_id should be 55234103
2017-11-05 21:59:30.772763 7f23302e4700 10 cephx server client.admin: start_session server_challenge 5c696d6cbed94d3c
2017-11-05 21:59:30.772771 7f23302e4700  2 mon.ceph03@2(peon) e14 send_reply 0x7f2342e9d4e0 0x7f234296d900 auth_reply(proto 2 0 (0) Success) v1
2017-11-05 21:59:30.773238 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342933600 for client.? 10.10.1.21:0/1317201829
2017-11-05 21:59:30.773247 7f23302e4700 20 mon.ceph03@2(peon) e14  caps
2017-11-05 21:59:30.773250 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(auth 26252..26332) dispatch 0x7f234296d900 auth(proto 2 32 bytes epoch 0) v1 from client.? 10.10.1.21:0/1317201829 con 0x7f2342f07800
2017-11-05 21:59:30.773256 7f23302e4700  5 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907754) is_readable = 1 - now=2017-11-05 21:59:30.773257 lease_expire=2017-11-05 21:59:35.342590 has v0 lc 112907754
2017-11-05 21:59:30.773266 7f23302e4700 10 mon.ceph03@2(peon).auth v26332 preprocess_query auth(proto 2 32 bytes epoch 0) v1 from client.? 10.10.1.21:0/1317201829
2017-11-05 21:59:30.773271 7f23302e4700 10 mon.ceph03@2(peon).auth v26332 prep_auth() blob_size=32
2017-11-05 21:59:30.773274 7f23302e4700 10 cephx server client.admin: handle_request get_auth_session_key for client.admin
2017-11-05 21:59:30.773302 7f23302e4700 20 cephx server client.admin:  checking key: req.key=371798402e1722a2 expected_key=371798402e1722a2
2017-11-05 21:59:30.773355 7f23302e4700 10 cephx: build_service_ticket_reply encoding 1 tickets with secret AQAh1Z5VjAN3ARAAw+R6HvdEfcFucLX1YSmG4Q==
2017-11-05 21:59:30.773379 7f23302e4700 10 cephx: build_service_ticket service auth secret_id 1699 ticket_info.ticket.name=client.admin
2017-11-05 21:59:30.773394 7f23302e4700 10 cephx keyserverdata: get_caps: name=client.admin
2017-11-05 21:59:30.773399 7f23302e4700 10 cephx keyserverdata: get_secret: num of caps=4
2017-11-05 21:59:30.773447 7f23302e4700  2 mon.ceph03@2(peon) e14 send_reply 0x7f2342e9d4e0 0x7f2342f14500 auth_reply(proto 2 0 (0) Success) v1
2017-11-05 21:59:30.773958 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342933600 for client.? 10.10.1.21:0/1317201829
2017-11-05 21:59:30.773969 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
2017-11-05 21:59:30.773972 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(auth 26252..26332) dispatch 0x7f2342f14500 auth(proto 2 165 bytes epoch 0) v1 from client.? 10.10.1.21:0/1317201829 con 0x7f2342f07800
2017-11-05 21:59:30.773979 7f23302e4700  5 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907754) is_readable = 1 - now=2017-11-05 21:59:30.773979 lease_expire=2017-11-05 21:59:35.342590 has v0 lc 112907754
2017-11-05 21:59:30.773988 7f23302e4700 10 mon.ceph03@2(peon).auth v26332 preprocess_query auth(proto 2 165 bytes epoch 0) v1 from client.? 10.10.1.21:0/1317201829
2017-11-05 21:59:30.773995 7f23302e4700 10 mon.ceph03@2(peon).auth v26332 prep_auth() blob_size=165
2017-11-05 21:59:30.773999 7f23302e4700 10 cephx server client.admin: handle_request get_principal_session_key
2017-11-05 21:59:30.774002 7f23302e4700 10 cephx: verify_authorizer decrypted service auth secret_id=1699
2017-11-05 21:59:30.774054 7f23302e4700 10 cephx: verify_authorizer global_id=55234103
2017-11-05 21:59:30.774080 7f23302e4700 10 cephx: verify_authorizer ok nonce 643c986966334873 reply_bl.length()=36
2017-11-05 21:59:30.774083 7f23302e4700 10 cephx server client.admin:  ticket_req.keys = 21
2017-11-05 21:59:30.774085 7f23302e4700 10 cephx server client.admin:  adding key for service mon
2017-11-05 21:59:30.774171 7f23302e4700 10 cephx server client.admin:  adding key for service osd
2017-11-05 21:59:30.774207 7f23302e4700 10 cephx keyserverdata: get_caps: name=client.admin
2017-11-05 21:59:30.774214 7f23302e4700 10 cephx keyserverdata: get_secret: num of caps=4
2017-11-05 21:59:30.774218 7f23302e4700 10 cephx server client.admin:  adding key for service mgr
2017-11-05 21:59:30.774252 7f23302e4700 10 cephx keyserverdata: get_caps: name=client.admin
2017-11-05 21:59:30.774258 7f23302e4700 10 cephx keyserverdata: get_secret: num of caps=4
2017-11-05 21:59:30.774270 7f23302e4700 10 cephx: build_service_ticket_reply encoding 3 tickets with secret AQCye/9ZNCEYLhAAM+olmzsar3Xp5cgMvcXkxg==
2017-11-05 21:59:30.774296 7f23302e4700 10 cephx: build_service_ticket service mon secret_id 20359 ticket_info.ticket.name=client.admin
2017-11-05 21:59:30.774323 7f23302e4700 10 cephx: build_service_ticket service osd secret_id 20359 ticket_info.ticket.name=client.admin
2017-11-05 21:59:30.774347 7f23302e4700 10 cephx: build_service_ticket service mgr secret_id 7 ticket_info.ticket.name=client.admin
2017-11-05 21:59:30.774381 7f23302e4700  2 mon.ceph03@2(peon) e14 send_reply 0x7f2342e9d4e0 0x7f234296d900 auth_reply(proto 2 0 (0) Success) v1
2017-11-05 21:59:30.774551 7f23302e4700 10 mon.ceph03@2(peon) e14 ms_handle_reset 0x7f2342f07800 10.10.1.21:0/1317201829
2017-11-05 21:59:30.774563 7f23302e4700 10 mon.ceph03@2(peon) e14 reset/close on session client.? 10.10.1.21:0/1317201829
2017-11-05 21:59:30.774568 7f23302e4700 10 mon.ceph03@2(peon) e14 remove_session 0x7f2342933600 client.? 10.10.1.21:0/1317201829 features 0x1ffddff8eea4fffb
2017-11-05 21:59:31.374295 7f232c2dc700 10 _calc_signature seq 248515282 front_crc_ = 4013096043 middle_crc = 0 data_crc = 0 sig = 3389298517379725511
2017-11-05 21:59:31.374391 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
2017-11-05 21:59:31.374403 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
2017-11-05 21:59:31.374407 7f23302e4700 20 is_capable service=mon command= read on cap allow *
2017-11-05 21:59:31.374411 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:31.374412 7f23302e4700 20  allow all
2017-11-05 21:59:31.374414 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
2017-11-05 21:59:31.374419 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:31.374420 7f23302e4700 20  allow all
2017-11-05 21:59:31.374430 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907754) handle_begin paxos(begin lc 112907754 fc 0 pn 13700 opn 0) v4
2017-11-05 21:59:31.374436 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos updating c 112907080..112907754) accepting value for 112907755 pn 13700
2017-11-05 21:59:31.411790 7f232c2dc700 10 _calc_signature seq 44 front_crc_ = 1766181624 middle_crc = 0 data_crc = 0 sig = 15812918676843081859
2017-11-05 21:59:31.411807 7f232c2dc700 20 Putting signature in client message(seq # 44): sig = 15812918676843081859
2017-11-05 21:59:31.440764 7f232c2dc700 10 _calc_signature seq 248515283 front_crc_ = 4114550218 middle_crc = 0 data_crc = 0 sig = 10113114477147680219
2017-11-05 21:59:31.440844 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
2017-11-05 21:59:31.440857 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
2017-11-05 21:59:31.440861 7f23302e4700 20 is_capable service=mon command= read on cap allow *
2017-11-05 21:59:31.440863 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:31.440865 7f23302e4700 20  allow all
2017-11-05 21:59:31.440867 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
2017-11-05 21:59:31.440868 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:31.440869 7f23302e4700 20  allow all
2017-11-05 21:59:31.440874 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos updating c 112907080..112907754) handle_commit on 112907755
2017-11-05 21:59:31.440879 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos updating c 112907080..112907755) store_state [112907755..112907755]
2017-11-05 21:59:31.441178 7f232c2dc700 10 _calc_signature seq 248515284 front_crc_ = 843291185 middle_crc = 0 data_crc = 0 sig = 2691046791697793914
2017-11-05 21:59:31.478345 7f23302e4700 10 mon.ceph03@2(peon) e14 refresh_from_paxos
2017-11-05 21:59:31.478382 7f23302e4700 10 mon.ceph03@2(peon) e14 refresh_from_paxos no cluster_fingerprint
2017-11-05 21:59:31.478418 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(pgmap 56600067..56600783) refresh
2017-11-05 21:59:31.478464 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mdsmap 1..1) refresh
2017-11-05 21:59:31.478514 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(osdmap 503..1105) refresh
2017-11-05 21:59:31.478552 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(logm 55596915..55597638) refresh
2017-11-05 21:59:31.478559 7f23302e4700 10 mon.ceph03@2(peon).log v55597638 update_from_paxos
2017-11-05 21:59:31.478562 7f23302e4700 10 mon.ceph03@2(peon).log v55597638 update_from_paxos version 55597638 summary v 55597637
2017-11-05 21:59:31.478572 7f23302e4700 10 mon.ceph03@2(peon).log v55597638 update_from_paxos latest full 55597637
2017-11-05 21:59:31.478596 7f23302e4700  7 mon.ceph03@2(peon).log v55597638 update_from_paxos applying incremental log 55597638 2017-11-05 21:59:30.880506 mon.ceph01mon mon.0 10.10.1.21:6789/0 867 : audit [DBG] from='client.? 10.10.1.21:0/1168097192' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch
2017-11-05 21:59:31.478620 7f23302e4700 20 mon.ceph03@2(peon).log v55597638 update_from_paxos logging for channel 'audit' to file '/var/log/ceph/ceph.audit.log'
2017-11-05 21:59:31.478658 7f23302e4700  7 mon.ceph03@2(peon).log v55597638 update_from_paxos applying incremental log 55597638 2017-11-05 21:59:30.929533 mon.ceph02mon mon.1 10.10.1.22:6789/0 363 : audit [DBG] from='client.? 10.10.1.21:0/1317201829' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch
2017-11-05 21:59:31.478674 7f23302e4700 20 mon.ceph03@2(peon).log v55597638 update_from_paxos logging for channel 'audit' to file '/var/log/ceph/ceph.audit.log'
2017-11-05 21:59:31.478682 7f23302e4700  7 mon.ceph03@2(peon).log v55597638 update_from_paxos applying incremental log 55597638 2017-11-05 21:59:31.014549 mon.ceph01mon mon.0 10.10.1.21:6789/0 868 : audit [DBG] from='client.? 10.10.1.21:0/1003889937' entity='client.admin' cmd=[{"prefix": "df", "format": "json"}]: dispatch
2017-11-05 21:59:31.478689 7f23302e4700 20 mon.ceph03@2(peon).log v55597638 update_from_paxos logging for channel 'audit' to file '/var/log/ceph/ceph.audit.log'
2017-11-05 21:59:31.478702 7f23302e4700 15 mon.ceph03@2(peon).log v55597638 update_from_paxos logging for 0 channels
2017-11-05 21:59:31.478709 7f23302e4700 10 mon.ceph03@2(peon).log v55597638 check_subs
2017-11-05 21:59:31.478762 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(monmap 1..14) refresh
2017-11-05 21:59:31.478824 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(auth 26252..26332) refresh
2017-11-05 21:59:31.478832 7f23302e4700 10 mon.ceph03@2(peon).auth v26332 update_from_paxos
2017-11-05 21:59:31.478881 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgr 1..394) refresh
2017-11-05 21:59:31.478938 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgrstat 56609513..56609643) refresh
2017-11-05 21:59:31.478945 7f23302e4700 10 mon.ceph03@2(peon).mgrstat  56609643
2017-11-05 21:59:31.479003 7f23302e4700 10 mon.ceph03@2(peon).mgrstat update_from_paxos v56609643 service_map e1
2017-11-05 21:59:31.479017 7f23302e4700 10 mon.ceph03@2(peon).mgrstat check_subs
2017-11-05 21:59:31.479020 7f23302e4700 20 mon.ceph03@2(peon).mgrstat update_logger
2017-11-05 21:59:31.479056 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(health 1..24) refresh
2017-11-05 21:59:31.479067 7f23302e4700 10 mon.ceph03@2(peon).health update_from_paxos
2017-11-05 21:59:31.479084 7f23302e4700 20 mon.ceph03@2(peon).health dump:{
    "quorum_health": {},
    "leader_health": {}
}

2017-11-05 21:59:31.479100 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(pgmap 56600067..56600783) post_refresh
2017-11-05 21:59:31.479105 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mdsmap 1..1) post_refresh
2017-11-05 21:59:31.479107 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(osdmap 503..1105) post_refresh
2017-11-05 21:59:31.479109 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(logm 55596915..55597638) post_refresh
2017-11-05 21:59:31.479111 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(monmap 1..14) post_refresh
2017-11-05 21:59:31.479112 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(auth 26252..26332) post_refresh
2017-11-05 21:59:31.479114 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgr 1..394) post_refresh
2017-11-05 21:59:31.479116 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgrstat 56609513..56609643) post_refresh
2017-11-05 21:59:31.479117 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(health 1..24) post_refresh
2017-11-05 21:59:31.479217 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
2017-11-05 21:59:31.479226 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
2017-11-05 21:59:31.479229 7f23302e4700 20 is_capable service=mon command= read on cap allow *
2017-11-05 21:59:31.479232 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:31.479233 7f23302e4700 20  allow all
2017-11-05 21:59:31.479235 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
2017-11-05 21:59:31.479236 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:31.479237 7f23302e4700 20  allow all
2017-11-05 21:59:31.479242 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907755) handle_lease on 112907755 now 2017-11-05 21:59:36.439247
2017-11-05 21:59:31.479272 7f23302e4700 20 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907755) reset_lease_timeout - setting timeout event
2017-11-05 21:59:31.479280 7f23302e4700  5 mon.ceph03@2(peon).monmap v14 apply_mon_features wait for service to be writeable
2017-11-05 21:59:31.479283 7f23302e4700  5 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907755) is_readable = 1 - now=2017-11-05 21:59:31.479284 lease_expire=2017-11-05 21:59:36.439247 has v0 lc 112907755
2017-11-05 21:59:31.479338 7f232c2dc700 10 _calc_signature seq 45 front_crc_ = 3341093255 middle_crc = 0 data_crc = 0 sig = 199787768109639415
2017-11-05 21:59:31.479354 7f232c2dc700 20 Putting signature in client message(seq # 45): sig = 199787768109639415
2017-11-05 21:59:32.481784 7f232c2dc700 10 _calc_signature seq 248515285 front_crc_ = 2658085271 middle_crc = 0 data_crc = 0 sig = 15694764974702500250
2017-11-05 21:59:32.481906 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
2017-11-05 21:59:32.481916 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
2017-11-05 21:59:32.481921 7f23302e4700 20 is_capable service=mon command= read on cap allow *
2017-11-05 21:59:32.481924 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:32.481925 7f23302e4700 20  allow all
2017-11-05 21:59:32.481930 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
2017-11-05 21:59:32.481932 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:32.481933 7f23302e4700 20  allow all
2017-11-05 21:59:32.481939 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907755) handle_begin paxos(begin lc 112907755 fc 0 pn 13700 opn 0) v4
2017-11-05 21:59:32.481944 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos updating c 112907080..112907755) accepting value for 112907756 pn 13700
2017-11-05 21:59:32.519206 7f232c2dc700 10 _calc_signature seq 46 front_crc_ = 916056675 middle_crc = 0 data_crc = 0 sig = 3319636971414846998
2017-11-05 21:59:32.519222 7f232c2dc700 20 Putting signature in client message(seq # 46): sig = 3319636971414846998
2017-11-05 21:59:32.548470 7f232c2dc700 10 _calc_signature seq 248515286 front_crc_ = 3380664477 middle_crc = 0 data_crc = 0 sig = 16252625642398161639
2017-11-05 21:59:32.548580 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
2017-11-05 21:59:32.548589 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
2017-11-05 21:59:32.548592 7f23302e4700 20 is_capable service=mon command= read on cap allow *
2017-11-05 21:59:32.548595 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:32.548596 7f23302e4700 20  allow all
2017-11-05 21:59:32.548598 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
2017-11-05 21:59:32.548600 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:32.548601 7f23302e4700 20  allow all
2017-11-05 21:59:32.548606 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos updating c 112907080..112907755) handle_commit on 112907756
2017-11-05 21:59:32.548611 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos updating c 112907080..112907756) store_state [112907756..112907756]
2017-11-05 21:59:32.548902 7f232c2dc700 10 _calc_signature seq 248515287 front_crc_ = 2140225877 middle_crc = 0 data_crc = 0 sig = 17236814342560090484
2017-11-05 21:59:32.581973 7f232c2dc700 10 _calc_signature seq 248515288 front_crc_ = 596978514 middle_crc = 0 data_crc = 0 sig = 17084270481735144864
2017-11-05 21:59:32.585752 7f23302e4700 10 mon.ceph03@2(peon) e14 refresh_from_paxos
2017-11-05 21:59:32.585791 7f23302e4700 10 mon.ceph03@2(peon) e14 refresh_from_paxos no cluster_fingerprint
2017-11-05 21:59:32.585826 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(pgmap 56600067..56600783) refresh
2017-11-05 21:59:32.585878 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mdsmap 1..1) refresh
2017-11-05 21:59:32.585937 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(osdmap 503..1105) refresh
2017-11-05 21:59:32.585980 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(logm 55596915..55597638) refresh
2017-11-05 21:59:32.585987 7f23302e4700 10 mon.ceph03@2(peon).log v55597638 update_from_paxos
2017-11-05 21:59:32.585990 7f23302e4700 10 mon.ceph03@2(peon).log v55597638 update_from_paxos version 55597638 summary v 55597638
2017-11-05 21:59:32.586037 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(monmap 1..14) refresh
2017-11-05 21:59:32.586064 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(auth 26252..26332) refresh
2017-11-05 21:59:32.586070 7f23302e4700 10 mon.ceph03@2(peon).auth v26332 update_from_paxos
2017-11-05 21:59:32.586118 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgr 1..394) refresh
2017-11-05 21:59:32.586206 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgrstat 56609513..56609644) refresh
2017-11-05 21:59:32.586215 7f23302e4700 10 mon.ceph03@2(peon).mgrstat  56609644
2017-11-05 21:59:32.586264 7f23302e4700 10 mon.ceph03@2(peon).mgrstat update_from_paxos v56609644 service_map e1
2017-11-05 21:59:32.586274 7f23302e4700 10 mon.ceph03@2(peon).mgrstat check_subs
2017-11-05 21:59:32.586276 7f23302e4700 20 mon.ceph03@2(peon).mgrstat update_logger
2017-11-05 21:59:32.586310 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(health 1..24) refresh
2017-11-05 21:59:32.586317 7f23302e4700 10 mon.ceph03@2(peon).health update_from_paxos
2017-11-05 21:59:32.586337 7f23302e4700 20 mon.ceph03@2(peon).health dump:{
    "quorum_health": {},
    "leader_health": {}
}

2017-11-05 21:59:32.586353 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(pgmap 56600067..56600783) post_refresh
2017-11-05 21:59:32.586356 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mdsmap 1..1) post_refresh
2017-11-05 21:59:32.586359 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(osdmap 503..1105) post_refresh
2017-11-05 21:59:32.586360 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(logm 55596915..55597638) post_refresh
2017-11-05 21:59:32.586362 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(monmap 1..14) post_refresh
2017-11-05 21:59:32.586364 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(auth 26252..26332) post_refresh
2017-11-05 21:59:32.586365 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgr 1..394) post_refresh
2017-11-05 21:59:32.586367 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgrstat 56609513..56609644) post_refresh
2017-11-05 21:59:32.586369 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(health 1..24) post_refresh
2017-11-05 21:59:32.586474 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
2017-11-05 21:59:32.586486 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
2017-11-05 21:59:32.586489 7f23302e4700 20 is_capable service=mon command= read on cap allow *
2017-11-05 21:59:32.586492 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:32.586493 7f23302e4700 20  allow all
2017-11-05 21:59:32.586495 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
2017-11-05 21:59:32.586497 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:32.586498 7f23302e4700 20  allow all
2017-11-05 21:59:32.586502 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907756) handle_lease on 112907756 now 2017-11-05 21:59:37.547047
2017-11-05 21:59:32.586540 7f23302e4700 20 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907756) reset_lease_timeout - setting timeout event
2017-11-05 21:59:32.586547 7f23302e4700  5 mon.ceph03@2(peon).monmap v14 apply_mon_features wait for service to be writeable
2017-11-05 21:59:32.586551 7f23302e4700  5 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907756) is_readable = 1 - now=2017-11-05 21:59:32.586551 lease_expire=2017-11-05 21:59:37.547047 has v0 lc 112907756
2017-11-05 21:59:32.586577 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
2017-11-05 21:59:32.586586 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
2017-11-05 21:59:32.586588 7f23302e4700 20 is_capable service=mon command= read on cap allow *
2017-11-05 21:59:32.586590 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:32.586591 7f23302e4700 20  allow all
2017-11-05 21:59:32.586586 7f232c2dc700 10 _calc_signature seq 47 front_crc_ = 514726373 middle_crc = 0 data_crc = 0 sig = 14918821708796830444
2017-11-05 21:59:32.586592 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
2017-11-05 21:59:32.586594 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:32.586594 7f232c2dc700 20 Putting signature in client message(seq # 47): sig = 14918821708796830444
2017-11-05 21:59:32.586596 7f23302e4700 20  allow all
2017-11-05 21:59:32.586599 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907756) handle_begin paxos(begin lc 112907756 fc 0 pn 13700 opn 0) v4
2017-11-05 21:59:32.586619 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos updating c 112907080..112907756) accepting value for 112907757 pn 13700
2017-11-05 21:59:32.630246 7f232c2dc700 10 _calc_signature seq 48 front_crc_ = 3355026546 middle_crc = 0 data_crc = 0 sig = 10281745577978676838
2017-11-05 21:59:32.630261 7f232c2dc700 20 Putting signature in client message(seq # 48): sig = 10281745577978676838
2017-11-05 21:59:32.659832 7f232c2dc700 10 _calc_signature seq 248515289 front_crc_ = 3094126013 middle_crc = 0 data_crc = 0 sig = 18218305787367258982
2017-11-05 21:59:32.659945 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
2017-11-05 21:59:32.659957 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
2017-11-05 21:59:32.659960 7f23302e4700 20 is_capable service=mon command= read on cap allow *
2017-11-05 21:59:32.659963 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:32.659964 7f23302e4700 20  allow all
2017-11-05 21:59:32.659966 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
2017-11-05 21:59:32.659967 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:32.659969 7f23302e4700 20  allow all
2017-11-05 21:59:32.659974 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos updating c 112907080..112907756) handle_commit on 112907757
2017-11-05 21:59:32.659979 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos updating c 112907080..112907757) store_state [112907757..112907757]
2017-11-05 21:59:32.660040 7f232c2dc700 10 _calc_signature seq 248515290 front_crc_ = 2812660264 middle_crc = 0 data_crc = 0 sig = 6326750952372116244
2017-11-05 21:59:32.696757 7f23302e4700 10 mon.ceph03@2(peon) e14 refresh_from_paxos
2017-11-05 21:59:32.696792 7f23302e4700 10 mon.ceph03@2(peon) e14 refresh_from_paxos no cluster_fingerprint
2017-11-05 21:59:32.696839 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(pgmap 56600067..56600783) refresh
2017-11-05 21:59:32.696888 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mdsmap 1..1) refresh
2017-11-05 21:59:32.696935 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(osdmap 503..1105) refresh
2017-11-05 21:59:32.696977 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(logm 55596915..55597639) refresh
2017-11-05 21:59:32.696984 7f23302e4700 10 mon.ceph03@2(peon).log v55597639 update_from_paxos
2017-11-05 21:59:32.696986 7f23302e4700 10 mon.ceph03@2(peon).log v55597639 update_from_paxos version 55597639 summary v 55597638
2017-11-05 21:59:32.696997 7f23302e4700 10 mon.ceph03@2(peon).log v55597639 update_from_paxos latest full 55597638
2017-11-05 21:59:32.697035 7f23302e4700  7 mon.ceph03@2(peon).log v55597639 update_from_paxos applying incremental log 55597639 2017-11-05 21:59:30.742011 mgr.ceph01mgr client.54805773 10.10.1.21:0/1619113657 1819 : audit [DBG] from='client.55026414 10.10.1.21:0/3473720941' entity='client.admin' cmd=[{"prefix": "osd pool stats", "target": ["mgr", ""], "format": "json"}]: dispatch
2017-11-05 21:59:32.697059 7f23302e4700 20 mon.ceph03@2(peon).log v55597639 update_from_paxos logging for channel 'audit' to file '/var/log/ceph/ceph.audit.log'
2017-11-05 21:59:32.697081 7f23302e4700 15 mon.ceph03@2(peon).log v55597639 update_from_paxos logging for 0 channels
2017-11-05 21:59:32.697087 7f23302e4700 10 mon.ceph03@2(peon).log v55597639 check_subs
2017-11-05 21:59:32.697146 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(monmap 1..14) refresh
2017-11-05 21:59:32.697182 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(auth 26252..26332) refresh
2017-11-05 21:59:32.697188 7f23302e4700 10 mon.ceph03@2(peon).auth v26332 update_from_paxos
2017-11-05 21:59:32.697242 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgr 1..394) refresh
2017-11-05 21:59:32.697302 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgrstat 56609513..56609644) refresh
2017-11-05 21:59:32.697332 7f23302e4700 10 mon.ceph03@2(peon).mgrstat  56609644
2017-11-05 21:59:32.697393 7f23302e4700 10 mon.ceph03@2(peon).mgrstat update_from_paxos v56609644 service_map e1
2017-11-05 21:59:32.697400 7f23302e4700 10 mon.ceph03@2(peon).mgrstat check_subs
2017-11-05 21:59:32.697402 7f23302e4700 20 mon.ceph03@2(peon).mgrstat update_logger
2017-11-05 21:59:32.697447 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(health 1..24) refresh
2017-11-05 21:59:32.697454 7f23302e4700 10 mon.ceph03@2(peon).health update_from_paxos
2017-11-05 21:59:32.697480 7f23302e4700 20 mon.ceph03@2(peon).health dump:{
    "quorum_health": {},
    "leader_health": {}
}

2017-11-05 21:59:32.697494 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(pgmap 56600067..56600783) post_refresh
2017-11-05 21:59:32.697496 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mdsmap 1..1) post_refresh
2017-11-05 21:59:32.697498 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(osdmap 503..1105) post_refresh
2017-11-05 21:59:32.697500 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(logm 55596915..55597639) post_refresh
2017-11-05 21:59:32.697502 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(monmap 1..14) post_refresh
2017-11-05 21:59:32.697503 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(auth 26252..26332) post_refresh
2017-11-05 21:59:32.697505 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgr 1..394) post_refresh
2017-11-05 21:59:32.697506 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgrstat 56609513..56609644) post_refresh
2017-11-05 21:59:32.697508 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(health 1..24) post_refresh
2017-11-05 21:59:32.697619 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
2017-11-05 21:59:32.697628 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
2017-11-05 21:59:32.697631 7f23302e4700 20 is_capable service=mon command= read on cap allow *
2017-11-05 21:59:32.697634 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:32.697635 7f23302e4700 20  allow all
2017-11-05 21:59:32.697637 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
2017-11-05 21:59:32.697638 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:32.697639 7f23302e4700 20  allow all
2017-11-05 21:59:32.697644 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907757) handle_lease on 112907757 now 2017-11-05 21:59:37.658131
2017-11-05 21:59:32.697676 7f23302e4700 20 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907757) reset_lease_timeout - setting timeout event
2017-11-05 21:59:32.697682 7f23302e4700  5 mon.ceph03@2(peon).monmap v14 apply_mon_features wait for service to be writeable
2017-11-05 21:59:32.697685 7f23302e4700  5 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907757) is_readable = 1 - now=2017-11-05 21:59:32.697685 lease_expire=2017-11-05 21:59:37.658131 has v0 lc 112907757
2017-11-05 21:59:32.697709 7f2332ae9700 11 mon.ceph03@2(peon) e14 tick
2017-11-05 21:59:32.697722 7f232c2dc700 10 _calc_signature seq 49 front_crc_ = 2344320304 middle_crc = 0 data_crc = 0 sig = 13886427265203894317
2017-11-05 21:59:32.697728 7f2332ae9700 10 mon.ceph03@2(peon).mds e1
2017-11-05 21:59:32.697735 7f2332ae9700 10 mon.ceph03@2(peon).osd e1105 e1105: 9 total, 9 up, 9 in
2017-11-05 21:59:32.697740 7f2332ae9700 10 mon.ceph03@2(peon).log v55597639 log
2017-11-05 21:59:32.697743 7f2332ae9700 10 mon.ceph03@2(peon).auth v26332 auth
2017-11-05 21:59:32.697732 7f232c2dc700 20 Putting signature in client message(seq # 49): sig = 13886427265203894317
2017-11-05 21:59:32.697746 7f2332ae9700 10 mon.ceph03@2(peon).health tick
2017-11-05 21:59:32.697748 7f2332ae9700 20 mon.ceph03@2(peon).health check_member_health
2017-11-05 21:59:32.697826 7f2332ae9700 10 mon.ceph03@2(peon).health check_member_health avail 92% total 277 GB, used 5677 MB, avail 257 GB
2017-11-05 21:59:32.697836 7f2332ae9700 20 mon.ceph03@2(peon) e14 sync_trim_providers
2017-11-05 21:59:32.899187 7f232d2de700 10 _calc_signature seq 6 front_crc_ = 3036032456 middle_crc = 0 data_crc = 0 sig = 5815314937307354885
2017-11-05 21:59:32.899206 7f232d2de700 20 Putting signature in client message(seq # 6): sig = 5815314937307354885
2017-11-05 21:59:34.276228 7f232c2dc700 10 _calc_signature seq 248515291 front_crc_ = 1672536857 middle_crc = 0 data_crc = 0 sig = 7744206132391408314
2017-11-05 21:59:34.276380 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
2017-11-05 21:59:34.276390 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
2017-11-05 21:59:34.276394 7f23302e4700 20 is_capable service=mon command= read on cap allow *
2017-11-05 21:59:34.276398 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:34.276399 7f23302e4700 20  allow all
2017-11-05 21:59:34.276401 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
2017-11-05 21:59:34.276402 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:34.276403 7f23302e4700 20  allow all
2017-11-05 21:59:34.276409 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907757) handle_begin paxos(begin lc 112907757 fc 0 pn 13700 opn 0) v4
2017-11-05 21:59:34.276414 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos updating c 112907080..112907757) accepting value for 112907758 pn 13700
2017-11-05 21:59:34.313293 7f232c2dc700 10 _calc_signature seq 50 front_crc_ = 25195403 middle_crc = 0 data_crc = 0 sig = 14303817380959227799
2017-11-05 21:59:34.313312 7f232c2dc700 20 Putting signature in client message(seq # 50): sig = 14303817380959227799
2017-11-05 21:59:34.342963 7f232c2dc700 10 _calc_signature seq 248515292 front_crc_ = 2334251510 middle_crc = 0 data_crc = 0 sig = 3881307708667676196
2017-11-05 21:59:34.343077 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
2017-11-05 21:59:34.343087 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
2017-11-05 21:59:34.343090 7f23302e4700 20 is_capable service=mon command= read on cap allow *
2017-11-05 21:59:34.343093 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:34.343094 7f23302e4700 20  allow all
2017-11-05 21:59:34.343096 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
2017-11-05 21:59:34.343098 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:34.343099 7f23302e4700 20  allow all
2017-11-05 21:59:34.343104 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos updating c 112907080..112907757) handle_commit on 112907758
2017-11-05 21:59:34.343109 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos updating c 112907080..112907758) store_state [112907758..112907758]
2017-11-05 21:59:34.343246 7f232c2dc700 10 _calc_signature seq 248515293 front_crc_ = 3382032424 middle_crc = 0 data_crc = 0 sig = 13032620410930708538
2017-11-05 21:59:34.379800 7f23302e4700 10 mon.ceph03@2(peon) e14 refresh_from_paxos
2017-11-05 21:59:34.379838 7f23302e4700 10 mon.ceph03@2(peon) e14 refresh_from_paxos no cluster_fingerprint
2017-11-05 21:59:34.379874 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(pgmap 56600067..56600783) refresh
2017-11-05 21:59:34.379922 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mdsmap 1..1) refresh
2017-11-05 21:59:34.379973 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(osdmap 503..1105) refresh
2017-11-05 21:59:34.380020 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(logm 55596915..55597639) refresh
2017-11-05 21:59:34.380026 7f23302e4700 10 mon.ceph03@2(peon).log v55597639 update_from_paxos
2017-11-05 21:59:34.380029 7f23302e4700 10 mon.ceph03@2(peon).log v55597639 update_from_paxos version 55597639 summary v 55597639
2017-11-05 21:59:34.380077 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(monmap 1..14) refresh
2017-11-05 21:59:34.380104 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(auth 26252..26332) refresh
2017-11-05 21:59:34.380127 7f23302e4700 10 mon.ceph03@2(peon).auth v26332 update_from_paxos
2017-11-05 21:59:34.380179 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgr 1..394) refresh
2017-11-05 21:59:34.380243 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgrstat 56609513..56609645) refresh
2017-11-05 21:59:34.380250 7f23302e4700 10 mon.ceph03@2(peon).mgrstat  56609645
2017-11-05 21:59:34.380308 7f23302e4700 10 mon.ceph03@2(peon).mgrstat update_from_paxos v56609645 service_map e1
2017-11-05 21:59:34.380315 7f23302e4700 10 mon.ceph03@2(peon).mgrstat check_subs
2017-11-05 21:59:34.380317 7f23302e4700 20 mon.ceph03@2(peon).mgrstat update_logger
2017-11-05 21:59:34.380352 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(health 1..24) refresh
2017-11-05 21:59:34.380359 7f23302e4700 10 mon.ceph03@2(peon).health update_from_paxos
2017-11-05 21:59:34.380376 7f23302e4700 20 mon.ceph03@2(peon).health dump:{
    "quorum_health": {},
    "leader_health": {}
}

2017-11-05 21:59:34.380392 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(pgmap 56600067..56600783) post_refresh
2017-11-05 21:59:34.380394 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mdsmap 1..1) post_refresh
2017-11-05 21:59:34.380399 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(osdmap 503..1105) post_refresh
2017-11-05 21:59:34.380401 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(logm 55596915..55597639) post_refresh
2017-11-05 21:59:34.380402 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(monmap 1..14) post_refresh
2017-11-05 21:59:34.380404 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(auth 26252..26332) post_refresh
2017-11-05 21:59:34.380406 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgr 1..394) post_refresh
2017-11-05 21:59:34.380407 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgrstat 56609513..56609645) post_refresh
2017-11-05 21:59:34.380409 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(health 1..24) post_refresh
2017-11-05 21:59:34.380507 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
2017-11-05 21:59:34.380516 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
2017-11-05 21:59:34.380519 7f23302e4700 20 is_capable service=mon command= read on cap allow *
2017-11-05 21:59:34.380522 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:34.380523 7f23302e4700 20  allow all
2017-11-05 21:59:34.380524 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
2017-11-05 21:59:34.380526 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:34.380527 7f23302e4700 20  allow all
2017-11-05 21:59:34.380532 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907758) handle_lease on 112907758 now 2017-11-05 21:59:39.341409
2017-11-05 21:59:34.380564 7f23302e4700 20 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907758) reset_lease_timeout - setting timeout event
2017-11-05 21:59:34.380572 7f23302e4700  5 mon.ceph03@2(peon).monmap v14 apply_mon_features wait for service to be writeable
2017-11-05 21:59:34.380576 7f23302e4700  5 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907758) is_readable = 1 - now=2017-11-05 21:59:34.380576 lease_expire=2017-11-05 21:59:39.341409 has v0 lc 112907758
2017-11-05 21:59:34.380616 7f232c2dc700 10 _calc_signature seq 51 front_crc_ = 221321949 middle_crc = 0 data_crc = 0 sig = 13265552358249501418
2017-11-05 21:59:34.380633 7f232c2dc700 20 Putting signature in client message(seq # 51): sig = 13265552358249501418
2017-11-05 21:59:36.016618 7f232d2de700 10 mon.ceph03@2(peon) e14 ms_verify_authorizer 10.10.1.23:0/3191937035 client protocol 0
2017-11-05 21:59:36.016641 7f232d2de700 10 In get_auth_session_handler for protocol 0
2017-11-05 21:59:36.016852 7f23302e4700 10 mon.ceph03@2(peon) e14 _ms_dispatch new session 0x7f2342933600 MonSession(client.? 10.10.1.23:0/3191937035 is open ) features 0x1ffddff8eea4fffb
2017-11-05 21:59:36.016864 7f23302e4700 20 mon.ceph03@2(peon) e14  caps
2017-11-05 21:59:36.016867 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(auth 26252..26332) dispatch 0x7f234296d900 auth(proto 0 30 bytes epoch 0) v1 from client.? 10.10.1.23:0/3191937035 con 0x7f2342f06000
2017-11-05 21:59:36.016874 7f23302e4700  5 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907758) is_readable = 1 - now=2017-11-05 21:59:36.016874 lease_expire=2017-11-05 21:59:39.341409 has v0 lc 112907758
2017-11-05 21:59:36.016884 7f23302e4700 10 mon.ceph03@2(peon).auth v26332 preprocess_query auth(proto 0 30 bytes epoch 0) v1 from client.? 10.10.1.23:0/3191937035
2017-11-05 21:59:36.016889 7f23302e4700 10 mon.ceph03@2(peon).auth v26332 prep_auth() blob_size=30
2017-11-05 21:59:36.016900 7f23302e4700 10 mon.ceph03@2(peon).auth v26332 AuthMonitor::assign_global_id m=auth(proto 0 30 bytes epoch 0) v1 mon=2/3 last_allocated=55234103 max_global_id=55244096
2017-11-05 21:59:36.016904 7f23302e4700 10 mon.ceph03@2(peon).auth v26332 next_global_id should be 55234106
2017-11-05 21:59:36.016951 7f23302e4700 10 cephx server client.admin: start_session server_challenge 7f99fad6b5c5d101
2017-11-05 21:59:36.016960 7f23302e4700  2 mon.ceph03@2(peon) e14 send_reply 0x7f2342e9d4e0 0x7f2342f14500 auth_reply(proto 2 0 (0) Success) v1
2017-11-05 21:59:36.017428 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342933600 for client.? 10.10.1.23:0/3191937035
2017-11-05 21:59:36.017436 7f23302e4700 20 mon.ceph03@2(peon) e14  caps
2017-11-05 21:59:36.017438 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(auth 26252..26332) dispatch 0x7f2342f14500 auth(proto 2 32 bytes epoch 0) v1 from client.? 10.10.1.23:0/3191937035 con 0x7f2342f06000
2017-11-05 21:59:36.017443 7f23302e4700  5 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907758) is_readable = 1 - now=2017-11-05 21:59:36.017444 lease_expire=2017-11-05 21:59:39.341409 has v0 lc 112907758
2017-11-05 21:59:36.017450 7f23302e4700 10 mon.ceph03@2(peon).auth v26332 preprocess_query auth(proto 2 32 bytes epoch 0) v1 from client.? 10.10.1.23:0/3191937035
2017-11-05 21:59:36.017456 7f23302e4700 10 mon.ceph03@2(peon).auth v26332 prep_auth() blob_size=32
2017-11-05 21:59:36.017459 7f23302e4700 10 cephx server client.admin: handle_request get_auth_session_key for client.admin
2017-11-05 21:59:36.017492 7f23302e4700 20 cephx server client.admin:  checking key: req.key=df499f5ee9b702d1 expected_key=df499f5ee9b702d1
2017-11-05 21:59:36.017542 7f23302e4700 10 cephx: build_service_ticket_reply encoding 1 tickets with secret AQAh1Z5VjAN3ARAAw+R6HvdEfcFucLX1YSmG4Q==
2017-11-05 21:59:36.017565 7f23302e4700 10 cephx: build_service_ticket service auth secret_id 1699 ticket_info.ticket.name=client.admin
2017-11-05 21:59:36.017579 7f23302e4700 10 cephx keyserverdata: get_caps: name=client.admin
2017-11-05 21:59:36.017583 7f23302e4700 10 cephx keyserverdata: get_secret: num of caps=4
2017-11-05 21:59:36.017629 7f23302e4700  2 mon.ceph03@2(peon) e14 send_reply 0x7f2342e9d4e0 0x7f234296d900 auth_reply(proto 2 0 (0) Success) v1
2017-11-05 21:59:36.017999 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342933600 for client.? 10.10.1.23:0/3191937035
2017-11-05 21:59:36.018007 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
2017-11-05 21:59:36.018009 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(auth 26252..26332) dispatch 0x7f234296d900 auth(proto 2 165 bytes epoch 0) v1 from client.? 10.10.1.23:0/3191937035 con 0x7f2342f06000
2017-11-05 21:59:36.018014 7f23302e4700  5 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907758) is_readable = 1 - now=2017-11-05 21:59:36.018014 lease_expire=2017-11-05 21:59:39.341409 has v0 lc 112907758
2017-11-05 21:59:36.018021 7f23302e4700 10 mon.ceph03@2(peon).auth v26332 preprocess_query auth(proto 2 165 bytes epoch 0) v1 from client.? 10.10.1.23:0/3191937035
2017-11-05 21:59:36.018025 7f23302e4700 10 mon.ceph03@2(peon).auth v26332 prep_auth() blob_size=165
2017-11-05 21:59:36.018027 7f23302e4700 10 cephx server client.admin: handle_request get_principal_session_key
2017-11-05 21:59:36.018030 7f23302e4700 10 cephx: verify_authorizer decrypted service auth secret_id=1699
2017-11-05 21:59:36.018071 7f23302e4700 10 cephx: verify_authorizer global_id=55234106
2017-11-05 21:59:36.018091 7f23302e4700 10 cephx: verify_authorizer ok nonce 6b8b4567327b23c6 reply_bl.length()=36
2017-11-05 21:59:36.018096 7f23302e4700 10 cephx server client.admin:  ticket_req.keys = 21
2017-11-05 21:59:36.018097 7f23302e4700 10 cephx server client.admin:  adding key for service mon
2017-11-05 21:59:36.018134 7f23302e4700 10 cephx server client.admin:  adding key for service osd
2017-11-05 21:59:36.018160 7f23302e4700 10 cephx keyserverdata: get_caps: name=client.admin
2017-11-05 21:59:36.018166 7f23302e4700 10 cephx keyserverdata: get_secret: num of caps=4
2017-11-05 21:59:36.018169 7f23302e4700 10 cephx server client.admin:  adding key for service mgr
2017-11-05 21:59:36.018191 7f23302e4700 10 cephx keyserverdata: get_caps: name=client.admin
2017-11-05 21:59:36.018196 7f23302e4700 10 cephx keyserverdata: get_secret: num of caps=4
2017-11-05 21:59:36.018201 7f23302e4700 10 cephx: build_service_ticket_reply encoding 3 tickets with secret AQC4e/9ZZUwLARAA4eu3mAYpVCYcqDn+3vicJQ==
2017-11-05 21:59:36.018218 7f23302e4700 10 cephx: build_service_ticket service mon secret_id 20359 ticket_info.ticket.name=client.admin
2017-11-05 21:59:36.018240 7f23302e4700 10 cephx: build_service_ticket service osd secret_id 20359 ticket_info.ticket.name=client.admin
2017-11-05 21:59:36.018263 7f23302e4700 10 cephx: build_service_ticket service mgr secret_id 7 ticket_info.ticket.name=client.admin
2017-11-05 21:59:36.018291 7f23302e4700  2 mon.ceph03@2(peon) e14 send_reply 0x7f2342e9d4e0 0x7f2342f14500 auth_reply(proto 2 0 (0) Success) v1
2017-11-05 21:59:36.018786 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342933600 for client.? 10.10.1.23:0/3191937035
2017-11-05 21:59:36.018804 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
2017-11-05 21:59:36.018808 7f23302e4700 20 is_capable service=mon command= read on cap allow *
2017-11-05 21:59:36.018813 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:36.018814 7f23302e4700 20  allow all
2017-11-05 21:59:36.018818 7f23302e4700 10 mon.ceph03@2(peon) e14 handle_subscribe mon_subscribe({monmap=0+}) v2
2017-11-05 21:59:36.018832 7f23302e4700 10 mon.ceph03@2(peon).monmap v14 check_sub monmap next 0 have 14
2017-11-05 21:59:36.018880 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342933600 for client.? 10.10.1.23:0/3191937035
2017-11-05 21:59:36.018885 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
2017-11-05 21:59:36.018889 7f23302e4700 20 is_capable service=mon command= read on cap allow *
2017-11-05 21:59:36.018891 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:36.018892 7f23302e4700 20  allow all
2017-11-05 21:59:36.018893 7f23302e4700 10 mon.ceph03@2(peon) e14 handle_subscribe mon_subscribe({mgrmap=0+}) v2
2017-11-05 21:59:36.018899 7f23302e4700 20 mon.ceph03@2(peon).mgr e394 Sending map to subscriber 0x7f2342f06000 10.10.1.23:0/3191937035
2017-11-05 21:59:36.018926 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342933600 for client.? 10.10.1.23:0/3191937035
2017-11-05 21:59:36.018929 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
2017-11-05 21:59:36.018931 7f23302e4700 20 is_capable service=mon command= read on cap allow *
2017-11-05 21:59:36.018933 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:36.018934 7f23302e4700 20  allow all
2017-11-05 21:59:36.018935 7f23302e4700 10 mon.ceph03@2(peon) e14 handle_subscribe mon_subscribe({osdmap=0}) v2
2017-11-05 21:59:36.018938 7f23302e4700 20 is_capable service=osd command= read on cap allow *
2017-11-05 21:59:36.018940 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:36.018941 7f23302e4700 20  allow all
2017-11-05 21:59:36.018942 7f23302e4700 10 mon.ceph03@2(peon).osd e1105 check_osdmap_sub 0x7f2342dc2fc0 next 0 (onetime)
2017-11-05 21:59:36.024233 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342933600 for client.? 10.10.1.23:0/3191937035
2017-11-05 21:59:36.024242 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
2017-11-05 21:59:36.028374 7f23302e4700  2 mon.ceph03@2(peon) e14 send_reply 0x7f2342f11f80 0x7f2342f50900 mon_command_ack([{"prefix": "get_command_descriptions"}]=0  v0) v1
2017-11-05 21:59:36.167424 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342933600 for client.? 10.10.1.23:0/3191937035
2017-11-05 21:59:36.167435 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
2017-11-05 21:59:36.167472 7f23302e4700  0 mon.ceph03@2(peon) e14 handle_command mon_command({"prefix": "osd pool stats"} v 0) v1
2017-11-05 21:59:36.167521 7f23302e4700 20 is_capable service=osd command=osd pool stats read on cap allow *
2017-11-05 21:59:36.167528 7f23302e4700 20  allow so far , doing grant allow *
2017-11-05 21:59:36.167530 7f23302e4700 20  allow all
2017-11-05 21:59:36.167531 7f23302e4700 10 mon.ceph03@2(peon) e14 _allowed_command capable
2017-11-05 21:59:36.167544 7f23302e4700  0 log_channel(audit) log [DBG] : from='client.? 10.10.1.23:0/3191937035' entity='client.admin' cmd=[{"prefix": "osd pool stats"}]: dispatch
2017-11-05 21:59:36.167585 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(osdmap 503..1105) dispatch 0x7f2342f50900 mon_command({"prefix": "osd pool stats"} v 0) v1 from client.55234106 10.10.1.23:0/3191937035 con 0x7f2342f06000
2017-11-05 21:59:36.167596 7f23302e4700  5 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907758) is_readable = 1 - now=2017-11-05 21:59:36.167596 lease_expire=2017-11-05 21:59:39.341409 has v0 lc 112907758
2017-11-05 21:59:36.167618 7f23302e4700 10 mon.ceph03@2(peon).osd e1105 preprocess_query mon_command({"prefix": "osd pool stats"} v 0) v1 from client.55234106 10.10.1.23:0/3191937035
2017-11-05 21:59:36.173253 7f23302e4700 -1 *** Caught signal (Aborted) **
 in thread 7f23302e4700 thread_name:ms_dispatch

 ceph version 12.2.1 (3e7492b9ada8bdc9a5cd0feafd42fbca27f9c38e) luminous (stable)
 1: (()+0x8c5429) [0x7f233948c429]
 2: (()+0x10330) [0x7f2337f5f330]
 3: (gsignal()+0x37) [0x7f2336750c37]
 4: (abort()+0x148) [0x7f2336754028]
 5: (()+0x447c19) [0x7f233900ec19]
 6: (OSDMonitor::preprocess_command(boost::intrusive_ptr<MonOpRequest>)+0xd5e) [0x7f23390de96e]
 7: (OSDMonitor::preprocess_query(boost::intrusive_ptr<MonOpRequest>)+0x3d6) [0x7f23390e7846]
 8: (PaxosService::dispatch(boost::intrusive_ptr<MonOpRequest>)+0x841) [0x7f23390989b1]
 9: (Monitor::handle_command(boost::intrusive_ptr<MonOpRequest>)+0x1c69) [0x7f2338f7f069]
 10: (Monitor::dispatch_op(boost::intrusive_ptr<MonOpRequest>)+0x91f) [0x7f2338f849bf]
 11: (Monitor::_ms_dispatch(Message*)+0x7b9) [0x7f2338f85cb9]
 12: (Monitor::ms_dispatch(Message*)+0x23) [0x7f2338faefb3]
 13: (DispatchQueue::entry()+0x78b) [0x7f233943921b]
 14: (DispatchQueue::DispatchThread::entry()+0xd) [0x7f233924162d]
 15: (()+0x8184) [0x7f2337f57184]
 16: (clone()+0x6d) [0x7f2336817ffd]
 NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.

--- begin dump of recent events ---
 -2792> 2017-11-05 21:59:12.458992 7f2338baee00  5 asok(0x7f2342958380) register_command perfcounters_dump hook 0x7f23428e8170
 -2791> 2017-11-05 21:59:12.459005 7f2338baee00  5 asok(0x7f2342958380) register_command 1 hook 0x7f23428e8170
 -2790> 2017-11-05 21:59:12.459008 7f2338baee00  5 asok(0x7f2342958380) register_command perf dump hook 0x7f23428e8170
 -2789> 2017-11-05 21:59:12.459012 7f2338baee00  5 asok(0x7f2342958380) register_command perfcounters_schema hook 0x7f23428e8170
 -2788> 2017-11-05 21:59:12.459015 7f2338baee00  5 asok(0x7f2342958380) register_command perf histogram dump hook 0x7f23428e8170
 -2787> 2017-11-05 21:59:12.459038 7f2338baee00  5 asok(0x7f2342958380) register_command 2 hook 0x7f23428e8170
 -2786> 2017-11-05 21:59:12.459047 7f2338baee00  5 asok(0x7f2342958380) register_command perf schema hook 0x7f23428e8170
 -2785> 2017-11-05 21:59:12.459050 7f2338baee00  5 asok(0x7f2342958380) register_command perf histogram schema hook 0x7f23428e8170
 -2784> 2017-11-05 21:59:12.459061 7f2338baee00  5 asok(0x7f2342958380) register_command perf reset hook 0x7f23428e8170
 -2783> 2017-11-05 21:59:12.459065 7f2338baee00  5 asok(0x7f2342958380) register_command config show hook 0x7f23428e8170
 -2782> 2017-11-05 21:59:12.459072 7f2338baee00  5 asok(0x7f2342958380) register_command config help hook 0x7f23428e8170
 -2781> 2017-11-05 21:59:12.459077 7f2338baee00  5 asok(0x7f2342958380) register_command config set hook 0x7f23428e8170
 -2780> 2017-11-05 21:59:12.459084 7f2338baee00  5 asok(0x7f2342958380) register_command config get hook 0x7f23428e8170
 -2779> 2017-11-05 21:59:12.459088 7f2338baee00  5 asok(0x7f2342958380) register_command config diff hook 0x7f23428e8170
 -2778> 2017-11-05 21:59:12.459095 7f2338baee00  5 asok(0x7f2342958380) register_command config diff get hook 0x7f23428e8170
 -2777> 2017-11-05 21:59:12.459100 7f2338baee00  5 asok(0x7f2342958380) register_command log flush hook 0x7f23428e8170
 -2776> 2017-11-05 21:59:12.459106 7f2338baee00  5 asok(0x7f2342958380) register_command log dump hook 0x7f23428e8170
 -2775> 2017-11-05 21:59:12.459110 7f2338baee00  5 asok(0x7f2342958380) register_command log reopen hook 0x7f23428e8170
 -2774> 2017-11-05 21:59:12.459119 7f2338baee00  5 asok(0x7f2342958380) register_command dump_mempools hook 0x7f2342ae42e8
 -2773> 2017-11-05 21:59:12.467593 7f2338baee00  0 ceph version 12.2.1 (3e7492b9ada8bdc9a5cd0feafd42fbca27f9c38e) luminous (stable), process (unknown), pid 15193
 -2772> 2017-11-05 21:59:12.468352 7f2338baee00  0 pidfile_write: ignore empty --pid-file
 -2771> 2017-11-05 21:59:12.470972 7f2338baee00  5 asok(0x7f2342958380) init /var/run/ceph/ceph-mon.ceph03.asok
 -2770> 2017-11-05 21:59:12.470992 7f2338baee00  5 asok(0x7f2342958380) bind_and_listen /var/run/ceph/ceph-mon.ceph03.asok
 -2769> 2017-11-05 21:59:12.471031 7f2338baee00  5 asok(0x7f2342958380) register_command 0 hook 0x7f23428e40c0
 -2768> 2017-11-05 21:59:12.471049 7f2338baee00  5 asok(0x7f2342958380) register_command version hook 0x7f23428e40c0
 -2767> 2017-11-05 21:59:12.471060 7f2338baee00  5 asok(0x7f2342958380) register_command git_version hook 0x7f23428e40c0
 -2766> 2017-11-05 21:59:12.471064 7f2338baee00  5 asok(0x7f2342958380) register_command help hook 0x7f23428e82c0
 -2765> 2017-11-05 21:59:12.471075 7f2338baee00  5 asok(0x7f2342958380) register_command get_command_descriptions hook 0x7f23428e82b0
 -2764> 2017-11-05 21:59:12.471141 7f2334320700  5 asok(0x7f2342958380) entry start
 -2763> 2017-11-05 21:59:12.477930 7f2338baee00  0 load: jerasure load: lrc load: isa
 -2762> 2017-11-05 21:59:12.478102 7f2338baee00  0  set rocksdb option compression = kNoCompression
 -2761> 2017-11-05 21:59:12.478114 7f2338baee00  0  set rocksdb option write_buffer_size = 33554432
 -2760> 2017-11-05 21:59:12.478145 7f2338baee00  0  set rocksdb option compression = kNoCompression
 -2759> 2017-11-05 21:59:12.478152 7f2338baee00  0  set rocksdb option write_buffer_size = 33554432
 -2758> 2017-11-05 21:59:12.478341 7f2338baee00  4 rocksdb: RocksDB version: 5.4.0

 -2757> 2017-11-05 21:59:12.478353 7f2338baee00  4 rocksdb: Git sha rocksdb_build_git_sha:@0@
 -2756> 2017-11-05 21:59:12.478355 7f2338baee00  4 rocksdb: Compile date Sep 26 2017
 -2755> 2017-11-05 21:59:12.478361 7f2338baee00  4 rocksdb: DB SUMMARY

 -2754> 2017-11-05 21:59:12.478409 7f2338baee00  4 rocksdb: CURRENT file:  CURRENT

 -2753> 2017-11-05 21:59:12.478415 7f2338baee00  4 rocksdb: IDENTITY file:  IDENTITY

 -2752> 2017-11-05 21:59:12.478419 7f2338baee00  4 rocksdb: MANIFEST file:  MANIFEST-000057 size: 216 Bytes

 -2751> 2017-11-05 21:59:12.478422 7f2338baee00  4 rocksdb: SST files in /var/lib/ceph/mon/ceph-ceph03/store.db dir, Total Num: 3, files: 000051.sst 000053.sst 000056.sst

 -2750> 2017-11-05 21:59:12.478424 7f2338baee00  4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-ceph03/store.db: 000058.log size: 170108 ;

 -2749> 2017-11-05 21:59:12.478426 7f2338baee00  4 rocksdb:                         Options.error_if_exists: 0
 -2748> 2017-11-05 21:59:12.478428 7f2338baee00  4 rocksdb:                       Options.create_if_missing: 0
 -2747> 2017-11-05 21:59:12.478429 7f2338baee00  4 rocksdb:                         Options.paranoid_checks: 1
 -2746> 2017-11-05 21:59:12.478435 7f2338baee00  4 rocksdb:                                     Options.env: 0x7f2341cdb2e0
 -2745> 2017-11-05 21:59:12.478437 7f2338baee00  4 rocksdb:                                Options.info_log: 0x7f2342bab540
 -2744> 2017-11-05 21:59:12.478438 7f2338baee00  4 rocksdb:                          Options.max_open_files: -1
 -2743> 2017-11-05 21:59:12.478439 7f2338baee00  4 rocksdb:                Options.max_file_opening_threads: 16
 -2742> 2017-11-05 21:59:12.478445 7f2338baee00  4 rocksdb:                               Options.use_fsync: 0
 -2741> 2017-11-05 21:59:12.478446 7f2338baee00  4 rocksdb:                       Options.max_log_file_size: 0
 -2740> 2017-11-05 21:59:12.478447 7f2338baee00  4 rocksdb:                  Options.max_manifest_file_size: 18446744073709551615
 -2739> 2017-11-05 21:59:12.478448 7f2338baee00  4 rocksdb:                   Options.log_file_time_to_roll: 0
 -2738> 2017-11-05 21:59:12.478449 7f2338baee00  4 rocksdb:                       Options.keep_log_file_num: 1000
 -2737> 2017-11-05 21:59:12.478450 7f2338baee00  4 rocksdb:                    Options.recycle_log_file_num: 0
 -2736> 2017-11-05 21:59:12.478451 7f2338baee00  4 rocksdb:                         Options.allow_fallocate: 1
 -2735> 2017-11-05 21:59:12.478452 7f2338baee00  4 rocksdb:                        Options.allow_mmap_reads: 0
 -2734> 2017-11-05 21:59:12.478453 7f2338baee00  4 rocksdb:                       Options.allow_mmap_writes: 0
 -2733> 2017-11-05 21:59:12.478454 7f2338baee00  4 rocksdb:                        Options.use_direct_reads: 0
 -2732> 2017-11-05 21:59:12.478455 7f2338baee00  4 rocksdb:                        Options.use_direct_io_for_flush_and_compaction: 0
 -2731> 2017-11-05 21:59:12.478456 7f2338baee00  4 rocksdb:          Options.create_missing_column_families: 0
 -2730> 2017-11-05 21:59:12.478456 7f2338baee00  4 rocksdb:                              Options.db_log_dir:
 -2729> 2017-11-05 21:59:12.478457 7f2338baee00  4 rocksdb:                                 Options.wal_dir: /var/lib/ceph/mon/ceph-ceph03/store.db
 -2728> 2017-11-05 21:59:12.478459 7f2338baee00  4 rocksdb:                Options.table_cache_numshardbits: 6
 -2727> 2017-11-05 21:59:12.478459 7f2338baee00  4 rocksdb:                      Options.max_subcompactions: 1
 -2726> 2017-11-05 21:59:12.478469 7f2338baee00  4 rocksdb:                  Options.max_background_flushes: 1
 -2725> 2017-11-05 21:59:12.478470 7f2338baee00  4 rocksdb:                         Options.WAL_ttl_seconds: 0
 -2724> 2017-11-05 21:59:12.478470 7f2338baee00  4 rocksdb:                       Options.WAL_size_limit_MB: 0
 -2723> 2017-11-05 21:59:12.478471 7f2338baee00  4 rocksdb:             Options.manifest_preallocation_size: 4194304
 -2722> 2017-11-05 21:59:12.478472 7f2338baee00  4 rocksdb:                     Options.is_fd_close_on_exec: 1
 -2721> 2017-11-05 21:59:12.478473 7f2338baee00  4 rocksdb:                   Options.advise_random_on_open: 1
 -2720> 2017-11-05 21:59:12.478474 7f2338baee00  4 rocksdb:                    Options.db_write_buffer_size: 0
 -2719> 2017-11-05 21:59:12.478475 7f2338baee00  4 rocksdb:         Options.access_hint_on_compaction_start: 1
 -2718> 2017-11-05 21:59:12.478476 7f2338baee00  4 rocksdb:  Options.new_table_reader_for_compaction_inputs: 0
 -2717> 2017-11-05 21:59:12.478476 7f2338baee00  4 rocksdb:               Options.compaction_readahead_size: 0
 -2716> 2017-11-05 21:59:12.478477 7f2338baee00  4 rocksdb:           Options.random_access_max_buffer_size: 1048576
 -2715> 2017-11-05 21:59:12.478478 7f2338baee00  4 rocksdb:           Options.writable_file_max_buffer_size: 1048576
 -2714> 2017-11-05 21:59:12.478479 7f2338baee00  4 rocksdb:                      Options.use_adaptive_mutex: 0
 -2713> 2017-11-05 21:59:12.478480 7f2338baee00  4 rocksdb:                            Options.rate_limiter: (nil)
 -2712> 2017-11-05 21:59:12.478481 7f2338baee00  4 rocksdb:     Options.sst_file_manager.rate_bytes_per_sec: 0
 -2711> 2017-11-05 21:59:12.478482 7f2338baee00  4 rocksdb:                          Options.bytes_per_sync: 0
 -2710> 2017-11-05 21:59:12.478483 7f2338baee00  4 rocksdb:                      Options.wal_bytes_per_sync: 0
 -2709> 2017-11-05 21:59:12.478484 7f2338baee00  4 rocksdb:                       Options.wal_recovery_mode: 2
 -2708> 2017-11-05 21:59:12.478485 7f2338baee00  4 rocksdb:                  Options.enable_thread_tracking: 0
 -2707> 2017-11-05 21:59:12.478486 7f2338baee00  4 rocksdb:         Options.allow_concurrent_memtable_write: 1
 -2706> 2017-11-05 21:59:12.478487 7f2338baee00  4 rocksdb:      Options.enable_write_thread_adaptive_yield: 1
 -2705> 2017-11-05 21:59:12.478488 7f2338baee00  4 rocksdb:             Options.write_thread_max_yield_usec: 100
 -2704> 2017-11-05 21:59:12.478488 7f2338baee00  4 rocksdb:            Options.write_thread_slow_yield_usec: 3
 -2703> 2017-11-05 21:59:12.478489 7f2338baee00  4 rocksdb:                               Options.row_cache: None
 -2702> 2017-11-05 21:59:12.478495 7f2338baee00  4 rocksdb:                              Options.wal_filter: None
 -2701> 2017-11-05 21:59:12.478496 7f2338baee00  4 rocksdb:             Options.avoid_flush_during_recovery: 0
 -2700> 2017-11-05 21:59:12.478498 7f2338baee00  4 rocksdb:             Options.base_background_compactions: 1
 -2699> 2017-11-05 21:59:12.478498 7f2338baee00  4 rocksdb:             Options.max_background_compactions: 1
 -2698> 2017-11-05 21:59:12.478499 7f2338baee00  4 rocksdb:             Options.avoid_flush_during_shutdown: 0
 -2697> 2017-11-05 21:59:12.478500 7f2338baee00  4 rocksdb:             Options.delayed_write_rate : 16777216
 -2696> 2017-11-05 21:59:12.478501 7f2338baee00  4 rocksdb:             Options.max_total_wal_size: 0
 -2695> 2017-11-05 21:59:12.478502 7f2338baee00  4 rocksdb:             Options.delete_obsolete_files_period_micros: 21600000000
 -2694> 2017-11-05 21:59:12.478504 7f2338baee00  4 rocksdb:                   Options.stats_dump_period_sec: 600
 -2693> 2017-11-05 21:59:12.478505 7f2338baee00  4 rocksdb: Compression algorithms supported:
 -2692> 2017-11-05 21:59:12.478505 7f2338baee00  4 rocksdb:     Snappy supported: 0
 -2691> 2017-11-05 21:59:12.478506 7f2338baee00  4 rocksdb:     Zlib supported: 0
 -2690> 2017-11-05 21:59:12.478507 7f2338baee00  4 rocksdb:     Bzip supported: 0
 -2689> 2017-11-05 21:59:12.478508 7f2338baee00  4 rocksdb:     LZ4 supported: 0
 -2688> 2017-11-05 21:59:12.478509 7f2338baee00  4 rocksdb:     ZSTD supported: 0
 -2687> 2017-11-05 21:59:12.478510 7f2338baee00  4 rocksdb: Fast CRC32 supported: 0
 -2686> 2017-11-05 21:59:12.478678 7f2338baee00  4 rocksdb: [/build/ceph-12.2.1/src/rocksdb/db/version_set.cc:2609] Recovering from manifest file: MANIFEST-000057

 -2685> 2017-11-05 21:59:12.478777 7f2338baee00  4 rocksdb: [/build/ceph-12.2.1/src/rocksdb/db/column_family.cc:407] --------------- Options for column family [default]:

 -2684> 2017-11-05 21:59:12.478786 7f2338baee00  4 rocksdb:               Options.comparator: leveldb.BytewiseComparator
 -2683> 2017-11-05 21:59:12.478788 7f2338baee00  4 rocksdb:           Options.merge_operator:
 -2682> 2017-11-05 21:59:12.478789 7f2338baee00  4 rocksdb:        Options.compaction_filter: None
 -2681> 2017-11-05 21:59:12.478792 7f2338baee00  4 rocksdb:        Options.compaction_filter_factory: None
 -2680> 2017-11-05 21:59:12.478793 7f2338baee00  4 rocksdb:         Options.memtable_factory: SkipListFactory
 -2679> 2017-11-05 21:59:12.478800 7f2338baee00  4 rocksdb:            Options.table_factory: BlockBasedTable
 -2678> 2017-11-05 21:59:12.478827 7f2338baee00  4 rocksdb:            table_factory options:   flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x7f23428e4170)
  cache_index_and_filter_blocks: 1
  cache_index_and_filter_blocks_with_high_priority: 1
  pin_l0_filter_and_index_blocks_in_cache: 1
  index_type: 0
  hash_index_allow_collision: 1
  checksum: 1
  no_block_cache: 0
  block_cache: 0x7f23429a5598
  block_cache_name: LRUCache
  block_cache_options:
    capacity : 134217728
    num_shard_bits : 4
    strict_capacity_limit : 0
    high_pri_pool_ratio: 0.000
  block_cache_compressed: (nil)
  persistent_cache: (nil)
  block_size: 4096
  block_size_deviation: 10
  block_restart_interval: 16
  index_block_restart_interval: 1
  filter_policy: rocksdb.BuiltinBloomFilter
  whole_key_filtering: 1
  format_version: 2

 -2677> 2017-11-05 21:59:12.478832 7f2338baee00  4 rocksdb:        Options.write_buffer_size: 33554432
 -2676> 2017-11-05 21:59:12.478834 7f2338baee00  4 rocksdb:  Options.max_write_buffer_number: 2
 -2675> 2017-11-05 21:59:12.478835 7f2338baee00  4 rocksdb:          Options.compression: NoCompression
 -2674> 2017-11-05 21:59:12.478837 7f2338baee00  4 rocksdb:                  Options.bottommost_compression: Disabled
 -2673> 2017-11-05 21:59:12.478838 7f2338baee00  4 rocksdb:       Options.prefix_extractor: nullptr
 -2672> 2017-11-05 21:59:12.478839 7f2338baee00  4 rocksdb:   Options.memtable_insert_with_hint_prefix_extractor: nullptr
 -2671> 2017-11-05 21:59:12.478840 7f2338baee00  4 rocksdb:             Options.num_levels: 7
 -2670> 2017-11-05 21:59:12.478841 7f2338baee00  4 rocksdb:        Options.min_write_buffer_number_to_merge: 1
 -2669> 2017-11-05 21:59:12.478842 7f2338baee00  4 rocksdb:     Options.max_write_buffer_number_to_maintain: 0
 -2668> 2017-11-05 21:59:12.478843 7f2338baee00  4 rocksdb:            Options.compression_opts.window_bits: -14
 -2667> 2017-11-05 21:59:12.478843 7f2338baee00  4 rocksdb:                  Options.compression_opts.level: -1
 -2666> 2017-11-05 21:59:12.478844 7f2338baee00  4 rocksdb:               Options.compression_opts.strategy: 0
 -2665> 2017-11-05 21:59:12.478845 7f2338baee00  4 rocksdb:         Options.compression_opts.max_dict_bytes: 0
 -2664> 2017-11-05 21:59:12.478846 7f2338baee00  4 rocksdb:      Options.level0_file_num_compaction_trigger: 4
 -2663> 2017-11-05 21:59:12.478847 7f2338baee00  4 rocksdb:          Options.level0_slowdown_writes_trigger: 20
 -2662> 2017-11-05 21:59:12.478848 7f2338baee00  4 rocksdb:              Options.level0_stop_writes_trigger: 36
 -2661> 2017-11-05 21:59:12.478849 7f2338baee00  4 rocksdb:                   Options.target_file_size_base: 67108864
 -2660> 2017-11-05 21:59:12.478850 7f2338baee00  4 rocksdb:             Options.target_file_size_multiplier: 1
 -2659> 2017-11-05 21:59:12.478851 7f2338baee00  4 rocksdb:                Options.max_bytes_for_level_base: 268435456
 -2658> 2017-11-05 21:59:12.478852 7f2338baee00  4 rocksdb: Options.level_compaction_dynamic_level_bytes: 0
 -2657> 2017-11-05 21:59:12.478853 7f2338baee00  4 rocksdb:          Options.max_bytes_for_level_multiplier: 10.000000
 -2656> 2017-11-05 21:59:12.478856 7f2338baee00  4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1
 -2655> 2017-11-05 21:59:12.478858 7f2338baee00  4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1
 -2654> 2017-11-05 21:59:12.478859 7f2338baee00  4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1
 -2653> 2017-11-05 21:59:12.478860 7f2338baee00  4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1
 -2652> 2017-11-05 21:59:12.478861 7f2338baee00  4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1
 -2651> 2017-11-05 21:59:12.478862 7f2338baee00  4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1
 -2650> 2017-11-05 21:59:12.478863 7f2338baee00  4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1
 -2649> 2017-11-05 21:59:12.478864 7f2338baee00  4 rocksdb:       Options.max_sequential_skip_in_iterations: 8
 -2648> 2017-11-05 21:59:12.478865 7f2338baee00  4 rocksdb:                    Options.max_compaction_bytes: 1677721600
 -2647> 2017-11-05 21:59:12.478866 7f2338baee00  4 rocksdb:                        Options.arena_block_size: 4194304
 -2646> 2017-11-05 21:59:12.478867 7f2338baee00  4 rocksdb:   Options.soft_pending_compaction_bytes_limit: 68719476736
 -2645> 2017-11-05 21:59:12.478868 7f2338baee00  4 rocksdb:   Options.hard_pending_compaction_bytes_limit: 274877906944
 -2644> 2017-11-05 21:59:12.478877 7f2338baee00  4 rocksdb:       Options.rate_limit_delay_max_milliseconds: 100
 -2643> 2017-11-05 21:59:12.478878 7f2338baee00  4 rocksdb:                Options.disable_auto_compactions: 0
 -2642> 2017-11-05 21:59:12.478885 7f2338baee00  4 rocksdb:                         Options.compaction_style: kCompactionStyleLevel
 -2641> 2017-11-05 21:59:12.478891 7f2338baee00  4 rocksdb:                           Options.compaction_pri: kByCompensatedSize
 -2640> 2017-11-05 21:59:12.478903 7f2338baee00  4 rocksdb:  Options.compaction_options_universal.size_ratio: 1
 -2639> 2017-11-05 21:59:12.478904 7f2338baee00  4 rocksdb: Options.compaction_options_universal.min_merge_width: 2
 -2638> 2017-11-05 21:59:12.478905 7f2338baee00  4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295
 -2637> 2017-11-05 21:59:12.478906 7f2338baee00  4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200
 -2636> 2017-11-05 21:59:12.478907 7f2338baee00  4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1
 -2635> 2017-11-05 21:59:12.478908 7f2338baee00  4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824
 -2634> 2017-11-05 21:59:12.478909 7f2338baee00  4 rocksdb:                   Options.table_properties_collectors:
 -2633> 2017-11-05 21:59:12.478910 7f2338baee00  4 rocksdb:                   Options.inplace_update_support: 0
 -2632> 2017-11-05 21:59:12.478911 7f2338baee00  4 rocksdb:                 Options.inplace_update_num_locks: 10000
 -2631> 2017-11-05 21:59:12.478912 7f2338baee00  4 rocksdb:               Options.memtable_prefix_bloom_size_ratio: 0.000000
 -2630> 2017-11-05 21:59:12.478914 7f2338baee00  4 rocksdb:   Options.memtable_huge_page_size: 0
 -2629> 2017-11-05 21:59:12.478915 7f2338baee00  4 rocksdb:                           Options.bloom_locality: 0
 -2628> 2017-11-05 21:59:12.478916 7f2338baee00  4 rocksdb:                    Options.max_successive_merges: 0
 -2627> 2017-11-05 21:59:12.478917 7f2338baee00  4 rocksdb:                Options.optimize_filters_for_hits: 0
 -2626> 2017-11-05 21:59:12.478918 7f2338baee00  4 rocksdb:                Options.paranoid_file_checks: 0
 -2625> 2017-11-05 21:59:12.478919 7f2338baee00  4 rocksdb:                Options.force_consistency_checks: 0
 -2624> 2017-11-05 21:59:12.478920 7f2338baee00  4 rocksdb:                Options.report_bg_io_stats: 0
 -2623> 2017-11-05 21:59:12.479894 7f2338baee00  4 rocksdb: [/build/ceph-12.2.1/src/rocksdb/db/version_set.cc:2859] Recovered from manifest file:/var/lib/ceph/mon/ceph-ceph03/store.db/MANIFEST-000057 succeeded,manifest_file_number is 57, next_file_number is 59, last_sequence is 11766, log_number is 0,prev_log_number is 0,max_column_family is 0

 -2622> 2017-11-05 21:59:12.479908 7f2338baee00  4 rocksdb: [/build/ceph-12.2.1/src/rocksdb/db/version_set.cc:2867] Column family [default] (ID 0), log number is 56

 -2621> 2017-11-05 21:59:12.479982 7f2338baee00  4 rocksdb: EVENT_LOG_v1 {"time_micros": 1509915552479972, "job": 1, "event": "recovery_started", "log_files": [58]}
 -2620> 2017-11-05 21:59:12.479990 7f2338baee00  4 rocksdb: [/build/ceph-12.2.1/src/rocksdb/db/db_impl_open.cc:482] Recovering log #58 mode 2
 -2619> 2017-11-05 21:59:12.493508 7f2338baee00  5 rocksdb: [/build/ceph-12.2.1/src/rocksdb/db/db_impl_open.cc:815] [default] [WriteLevel0TableForRecovery] Level-0 table #59: started
 -2618> 2017-11-05 21:59:12.527095 7f2338baee00  4 rocksdb: EVENT_LOG_v1 {"time_micros": 1509915552527081, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 59, "file_size": 113806, "table_properties": {"data_size": 112012, "index_size": 500, "filter_size": 377, "raw_key_size": 901, "raw_average_key_size": 24, "raw_value_size": 110963, "raw_average_value_size": 2999, "num_data_blocks": 15, "num_entries": 37, "filter_policy_name": "rocksdb.BuiltinBloomFilter", "kDeletedKeys": "0", "kMergeOperands": "0"}}
 -2617> 2017-11-05 21:59:12.527113 7f2338baee00  5 rocksdb: [/build/ceph-12.2.1/src/rocksdb/db/db_impl_open.cc:847] [default] [WriteLevel0TableForRecovery] Level-0 table #59: 113806 bytes OK
 -2616> 2017-11-05 21:59:12.527135 7f2338baee00  4 rocksdb: [/build/ceph-12.2.1/src/rocksdb/db/version_set.cc:2395] Creating manifest 60

 -2615> 2017-11-05 21:59:12.617442 7f2338baee00  4 rocksdb: EVENT_LOG_v1 {"time_micros": 1509915552617437, "job": 1, "event": "recovery_finished"}
 -2614> 2017-11-05 21:59:12.617728 7f2338baee00  5 rocksdb: [/build/ceph-12.2.1/src/rocksdb/db/db_impl_files.cc:307] [JOB 2] Delete /var/lib/ceph/mon/ceph-ceph03/store.db//MANIFEST-000057 type=3 #57 -- OK

 -2613> 2017-11-05 21:59:12.617784 7f2338baee00  5 rocksdb: [/build/ceph-12.2.1/src/rocksdb/db/db_impl_files.cc:307] [JOB 2] Delete /var/lib/ceph/mon/ceph-ceph03/store.db//000058.log type=0 #58 -- OK

 -2612> 2017-11-05 21:59:12.685504 7f2338baee00  4 rocksdb: [/build/ceph-12.2.1/src/rocksdb/db/db_impl_open.cc:1063] DB pointer 0x7f2342ca0000
 -2611> 2017-11-05 21:59:12.686191 7f2338baee00 10 obtain_monmap
 -2610> 2017-11-05 21:59:12.686560 7f2338baee00 10 obtain_monmap read last committed monmap ver 14
 -2609> 2017-11-05 21:59:12.687187 7f232d2de700  2 Event(0x7f23431a0e80 nevent=5000 time_id=1).set_owner idx=2 owner=139789058565888
 -2608> 2017-11-05 21:59:12.687185 7f232c2dc700  2 Event(0x7f2342957a80 nevent=5000 time_id=1).set_owner idx=0 owner=139789041780480
 -2607> 2017-11-05 21:59:12.687192 7f232cadd700  2 Event(0x7f23431a1280 nevent=5000 time_id=1).set_owner idx=1 owner=139789050173184
 -2606> 2017-11-05 21:59:12.687317 7f2338baee00  0 starting mon.ceph03 rank 2 at public addr 10.10.1.23:6789/0 at bind addr 10.10.1.23:6789/0 mon_data /var/lib/ceph/mon/ceph-ceph03 fsid 726a42b3-ae4c-4d0b-82c5-0a9065711b5f
 -2605> 2017-11-05 21:59:12.687429 7f2338baee00  1 -- 10.10.1.23:6789/0 learned_addr learned my addr 10.10.1.23:6789/0
 -2604> 2017-11-05 21:59:12.687440 7f2338baee00  1 -- 10.10.1.23:6789/0 _finish_bind bind my_inst.addr is 10.10.1.23:6789/0
 -2603> 2017-11-05 21:59:12.687485 7f2338baee00  0 starting mon.ceph03 rank 2 at 10.10.1.23:6789/0 mon_data /var/lib/ceph/mon/ceph-ceph03 fsid 726a42b3-ae4c-4d0b-82c5-0a9065711b5f
 -2602> 2017-11-05 21:59:12.687515 7f2338baee00  5 adding auth protocol: cephx
 -2601> 2017-11-05 21:59:12.687522 7f2338baee00  5 adding auth protocol: cephx
 -2600> 2017-11-05 21:59:12.687561 7f2338baee00 10 log_channel(cluster) update_config to_monitors: true to_syslog: false syslog_facility: daemon prio: info to_graylog: false graylog_host: 127.0.0.1 graylog_port: 12201)
 -2599> 2017-11-05 21:59:12.687571 7f2338baee00 10 log_channel(audit) update_config to_monitors: true to_syslog: false syslog_facility: local0 prio: info to_graylog: false graylog_host: 127.0.0.1 graylog_port: 12201)
 -2598> 2017-11-05 21:59:12.688266 7f2338baee00  1 mon.ceph03@-1(probing) e14 preinit fsid 726a42b3-ae4c-4d0b-82c5-0a9065711b5f
 -2597> 2017-11-05 21:59:12.688405 7f2338baee00 10 mon.ceph03@-1(probing) e14 check_fsid cluster_uuid contains '726a42b3-ae4c-4d0b-82c5-0a9065711b5f'
 -2596> 2017-11-05 21:59:12.688436 7f2338baee00 10 mon.ceph03@-1(probing) e14 features compat={},rocompat={},incompat={1=initial feature set (~v.18),3=single paxos with k/v store (v0.?),4=support erasure code pools,5=new-style osdmap encoding,6=support isa/lrc erasure code,7=support shec erasure code,8=support monmap features,9=luminous ondisk layout}
 -2595> 2017-11-05 21:59:12.688451 7f2338baee00 10 mon.ceph03@-1(probing) e14 calc_quorum_requirements required_features 153140804152475648
 -2594> 2017-11-05 21:59:12.688455 7f2338baee00 10 mon.ceph03@-1(probing) e14 required_features 153140804152475648
 -2593> 2017-11-05 21:59:12.688471 7f2338baee00 10 mon.ceph03@-1(probing) e14 has_ever_joined = 1
 -2592> 2017-11-05 21:59:12.688526 7f2338baee00 10 mon.ceph03@-1(probing) e14 sync_last_committed_floor 0
 -2591> 2017-11-05 21:59:12.688532 7f2338baee00 10 mon.ceph03@-1(probing) e14 init_paxos
 -2590> 2017-11-05 21:59:12.688613 7f2338baee00 10 mon.ceph03@-1(probing).paxos(paxos recovering c 112907080..112907717) init last_pn: 12300 accepted_pn: 13500 last_committed: 112907717 first_committed: 112907080
 -2589> 2017-11-05 21:59:12.688635 7f2338baee00 10 mon.ceph03@-1(probing).paxos(paxos recovering c 112907080..112907717) init
 -2588> 2017-11-05 21:59:12.688657 7f2338baee00  1 mon.ceph03@-1(probing).mds e0 Unable to load 'last_metadata'
 -2587> 2017-11-05 21:59:12.688699 7f2338baee00 10 mon.ceph03@-1(probing).health init
 -2586> 2017-11-05 21:59:12.688705 7f2338baee00 10 mon.ceph03@-1(probing) e14 refresh_from_paxos
 -2585> 2017-11-05 21:59:12.688716 7f2338baee00 10 mon.ceph03@-1(probing) e14 refresh_from_paxos no cluster_fingerprint
 -2584> 2017-11-05 21:59:12.688770 7f2338baee00  1 mon.ceph03@-1(probing).paxosservice(pgmap 56600067..56600783) refresh upgraded, format 0 -> 1
 -2583> 2017-11-05 21:59:12.688779 7f2338baee00  1 mon.ceph03@-1(probing).pg v0 on_upgrade discarding in-core PGMap
 -2582> 2017-11-05 21:59:12.688792 7f2338baee00 10 mon.ceph03@-1(probing).paxosservice(pgmap 56600067..56600783) refresh
 -2581> 2017-11-05 21:59:12.688803 7f2338baee00 10 mon.ceph03@-1(probing).pg v0 update_from_paxos deleted, clearing in-memory PGMap
 -2580> 2017-11-05 21:59:12.688856 7f2338baee00 10 mon.ceph03@-1(probing).paxosservice(mdsmap 1..1) refresh
 -2579> 2017-11-05 21:59:12.688863 7f2338baee00 10 mon.ceph03@-1(probing).mds e0 update_from_paxos version 1, my e 0
 -2578> 2017-11-05 21:59:12.688888 7f2338baee00 10 mon.ceph03@-1(probing).mds e0 update_from_paxos got 1
 -2577> 2017-11-05 21:59:12.688917 7f2338baee00  4 mon.ceph03@-1(probing).mds e1 new map
 -2576> 2017-11-05 21:59:12.688922 7f2338baee00  0 mon.ceph03@-1(probing).mds e1 print_map
e1
enable_multiple, ever_enabled_multiple: 0,0
compat: compat={},rocompat={},incompat={}
legacy client fscid: -1

No filesystems configured

 -2575> 2017-11-05 21:59:12.688937 7f2338baee00 10 mon.ceph03@-1(probing).mds e1 update_logger
 -2574> 2017-11-05 21:59:12.689075 7f2338baee00 10 mon.ceph03@-1(probing).paxosservice(osdmap 503..1105) refresh
 -2573> 2017-11-05 21:59:12.689089 7f2338baee00 15 mon.ceph03@-1(probing).osd e0 update_from_paxos paxos e 1105, my e 0
 -2572> 2017-11-05 21:59:12.689173 7f2338baee00  7 mon.ceph03@-1(probing).osd e0 update_from_paxos loading latest full map e1105
 -2571> 2017-11-05 21:59:12.689357 7f2338baee00  7 mon.ceph03@-1(probing).osd e1105 update_from_paxos loading creating_pgs last_scan_epoch 1104 with 0 pgs
 -2570> 2017-11-05 21:59:12.689368 7f2338baee00 10 mon.ceph03@-1(probing).osd e1105 update_from_paxos pgservice is mgrstat
 -2569> 2017-11-05 21:59:12.689372 7f2338baee00 10 mon.ceph03@-1(probing).osd e1105 check_osdmap_subs
 -2568> 2017-11-05 21:59:12.689382 7f2338baee00 10 mon.ceph03@-1(probing).osd e1105 share_map_with_random_osd no up osd on our session map
 -2567> 2017-11-05 21:59:12.689385 7f2338baee00 10 mon.ceph03@-1(probing).osd e1105 update_logger
 -2566> 2017-11-05 21:59:12.689390 7f2338baee00  0 mon.ceph03@-1(probing).osd e1105 crush map has features 1107558400, adjusting msgr requires
 -2565> 2017-11-05 21:59:12.689393 7f2338baee00  0 mon.ceph03@-1(probing).osd e1105 crush map has features 1107558400, adjusting msgr requires
 -2564> 2017-11-05 21:59:12.689398 7f2338baee00  0 mon.ceph03@-1(probing).osd e1105 crush map has features 720575941486854144, adjusting msgr requires
 -2563> 2017-11-05 21:59:12.689413 7f2338baee00  0 mon.ceph03@-1(probing).osd e1105 crush map has features 1107558400, adjusting msgr requires
 -2562> 2017-11-05 21:59:12.689455 7f2338baee00 20 queue 0x7f2342d4ae00 1 [0,512)
 -2561> 2017-11-05 21:59:12.689462 7f2338baee00 20 queue 0x7f2342d4ae00 2 [0,128)
 -2560> 2017-11-05 21:59:12.689464 7f2338baee00 20 queue 0x7f2342d4ae00 4 [0,128)
 -2559> 2017-11-05 21:59:12.689465 7f2338baee00 10 mon.ceph03@-1(probing).osd e1105 start_mapping started mapping job 0x7f2342d4ae00 at 2017-11-05 21:59:12.689416
 -2558> 2017-11-05 21:59:12.689587 7f2338baee00 10 mon.ceph03@-1(probing).paxosservice(logm 55596915..55597627) refresh
 -2557> 2017-11-05 21:59:12.689595 7f2338baee00 10 mon.ceph03@-1(probing).log v55597627 update_from_paxos
 -2556> 2017-11-05 21:59:12.689598 7f2338baee00 10 mon.ceph03@-1(probing).log v55597627 update_from_paxos version 55597627 summary v 0
 -2555> 2017-11-05 21:59:12.689608 7f2338baee00 10 mon.ceph03@-1(probing).log v55597627 update_from_paxos latest full 55597626
 -2554> 2017-11-05 21:59:12.689664 7f2338baee00  7 mon.ceph03@-1(probing).log v55597627 update_from_paxos loading summary e55597626
 -2553> 2017-11-05 21:59:12.689819 7f2338baee00  7 mon.ceph03@-1(probing).log v55597627 update_from_paxos loaded summary e55597626
 -2552> 2017-11-05 21:59:12.689856 7f2338baee00  7 mon.ceph03@-1(probing).log v55597627 update_from_paxos applying incremental log 55597627 2017-11-05 21:58:22.490661 mgr.ceph01mgr client.54805773 10.10.1.21:0/1619113657 1809 : audit [DBG] from='client.55026351 10.10.1.23:0/1865493862' entity='client.admin' cmd=[{"prefix": "osd pool stats", "target": ["mgr", ""]}]: dispatch
 -2551> 2017-11-05 21:59:12.689880 7f2338baee00 20 mon.ceph03@-1(probing).log v55597627 update_from_paxos logging for channel 'audit' to file '/var/log/ceph/ceph.audit.log'
 -2550> 2017-11-05 21:59:12.689891 7f2338baee00  7 mon.ceph03@-1(probing).log v55597627 update_from_paxos applying incremental log 55597627 2017-11-05 21:58:25.101829 mgr.ceph01mgr client.54805773 10.10.1.21:0/1619113657 1810 : audit [DBG] from='mon.0 10.10.1.21:0/24116' entity='mon.' cmd=[{"prefix": "osd pool stats"}]: dispatch
 -2549> 2017-11-05 21:59:12.689901 7f2338baee00 20 mon.ceph03@-1(probing).log v55597627 update_from_paxos logging for channel 'audit' to file '/var/log/ceph/ceph.audit.log'
 -2548> 2017-11-05 21:59:12.689908 7f2338baee00  7 mon.ceph03@-1(probing).log v55597627 update_from_paxos applying incremental log 55597627 2017-11-05 21:58:26.489612 mgr.ceph01mgr client.54805773 10.10.1.21:0/1619113657 1811 : audit [DBG] from='client.55006306 10.10.1.21:0/3517645621' entity='client.admin' cmd=[{"prefix": "osd pool stats", "target": ["mgr", ""], "format": "json"}]: dispatch
 -2547> 2017-11-05 21:59:12.689915 7f2338baee00 20 mon.ceph03@-1(probing).log v55597627 update_from_paxos logging for channel 'audit' to file '/var/log/ceph/ceph.audit.log'
 -2546> 2017-11-05 21:59:12.689920 7f2338baee00  7 mon.ceph03@-1(probing).log v55597627 update_from_paxos applying incremental log 55597627 2017-11-05 21:58:30.371628 mgr.ceph01mgr client.54805773 10.10.1.21:0/1619113657 1812 : audit [DBG] from='client.55026357 10.10.1.23:0/435814924' entity='client.admin' cmd=[{"prefix": "osd pool stats", "target": ["mgr", ""]}]: dispatch
 -2545> 2017-11-05 21:59:12.689926 7f2338baee00 20 mon.ceph03@-1(probing).log v55597627 update_from_paxos logging for channel 'audit' to file '/var/log/ceph/ceph.audit.log'
 -2544> 2017-11-05 21:59:12.689933 7f2338baee00 15 mon.ceph03@-1(probing).log v55597627 update_from_paxos logging for 0 channels
 -2543> 2017-11-05 21:59:12.689935 7f2338baee00 10 mon.ceph03@-1(probing).log v55597627 check_subs
 -2542> 2017-11-05 21:59:12.689968 7f2338baee00 10 mon.ceph03@-1(probing).paxosservice(monmap 1..14) refresh
 -2541> 2017-11-05 21:59:12.690012 7f2338baee00  1 mon.ceph03@-1(probing).paxosservice(auth 26252..26331) refresh upgraded, format 0 -> 2
 -2540> 2017-11-05 21:59:12.690019 7f2338baee00 10 mon.ceph03@-1(probing).paxosservice(auth 26252..26331) refresh
 -2539> 2017-11-05 21:59:12.690022 7f2338baee00 10 mon.ceph03@-1(probing).auth v26331 update_from_paxos
 -2538> 2017-11-05 21:59:12.690048 7f2338baee00 10 mon.ceph03@-1(probing).auth v26331 update_from_paxos version 26331 keys ver 0 latest 26321
 -2537> 2017-11-05 21:59:12.690080 7f2338baee00  7 mon.ceph03@-1(probing).auth v26331 update_from_paxos loading summary e 26321
 -2536> 2017-11-05 21:59:12.690086 7f2338baee00  7 mon.ceph03@-1(probing).auth v26331 update_from_paxos latest length 3603
 -2535> 2017-11-05 21:59:12.690788 7f2338baee00 10 mon.ceph03@-1(probing).auth v26331 update_from_paxos key server version 26321
 -2534> 2017-11-05 21:59:12.690813 7f2338baee00 20 mon.ceph03@-1(probing).auth v26331 update_from_paxos walking through version 26322 len 19
 -2533> 2017-11-05 21:59:12.690830 7f2338baee00 20 mon.ceph03@-1(probing).auth v26331 update_from_paxos walking through version 26323 len 19
 -2532> 2017-11-05 21:59:12.690845 7f2338baee00 20 mon.ceph03@-1(probing).auth v26331 update_from_paxos walking through version 26324 len 19
 -2531> 2017-11-05 21:59:12.690859 7f2338baee00 20 mon.ceph03@-1(probing).auth v26331 update_from_paxos walking through version 26325 len 19
 -2530> 2017-11-05 21:59:12.690869 7f2338baee00 20 mon.ceph03@-1(probing).auth v26331 update_from_paxos walking through version 26326 len 19
 -2529> 2017-11-05 21:59:12.690882 7f2338baee00 20 mon.ceph03@-1(probing).auth v26331 update_from_paxos walking through version 26327 len 19
 -2528> 2017-11-05 21:59:12.690893 7f2338baee00 20 mon.ceph03@-1(probing).auth v26331 update_from_paxos walking through version 26328 len 19
 -2527> 2017-11-05 21:59:12.690971 7f2338baee00 20 mon.ceph03@-1(probing).auth v26331 update_from_paxos walking through version 26329 len 19
 -2526> 2017-11-05 21:59:12.691020 7f2338baee00 20 mon.ceph03@-1(probing).auth v26331 update_from_paxos walking through version 26330 len 19
 -2525> 2017-11-05 21:59:12.691034 7f2338baee00 20 mon.ceph03@-1(probing).auth v26331 update_from_paxos walking through version 26331 len 19
 -2524> 2017-11-05 21:59:12.691041 7f2338baee00 10 mon.ceph03@-1(probing).auth v26331 update_from_paxos() last_allocated_id=55234096 max_global_id=55234096 format_version 2
 -2523> 2017-11-05 21:59:12.691095 7f2338baee00 10 mon.ceph03@-1(probing).paxosservice(mgr 1..394) refresh
 -2522> 2017-11-05 21:59:12.691102 7f2338baee00  4 mon.ceph03@-1(probing).mgr e0 loading version 394
 -2521> 2017-11-05 21:59:12.691140 7f2338baee00  4 mon.ceph03@-1(probing).mgr e394 active server: 10.10.1.21:6806/22556(54805773)
 -2520> 2017-11-05 21:59:12.691176 7f2338baee00  4 mon.ceph03@-1(probing).mgr e394 mkfs or daemon transitioned to available, loading commands
 -2519> 2017-11-05 21:59:12.691190 7f2338baee00 -1 mon.ceph03@-1(probing).mgr e394 Failed to load mgr commands: (2) No such file or directory
 -2518> 2017-11-05 21:59:12.691212 7f2338baee00  4 mgrc handle_mgr_map Got map version 394
 -2517> 2017-11-05 21:59:12.691215 7f2338baee00  4 mgrc handle_mgr_map Active mgr is now 10.10.1.21:6806/22556
 -2516> 2017-11-05 21:59:12.691219 7f2338baee00  4 mgrc reconnect Starting new session with 10.10.1.21:6806/22556
 -2515> 2017-11-05 21:59:12.691243 7f2338baee00  1 -- - --> 10.10.1.21:6806/22556 -- mgropen(unknown.ceph03) v2 -- 0x7f234296d900 con 0
 -2514> 2017-11-05 21:59:12.691296 7f2338baee00 10 mon.ceph03@-1(probing).paxosservice(mgrstat 56609513..56609621) refresh
 -2513> 2017-11-05 21:59:12.691304 7f2338baee00 10 mon.ceph03@-1(probing).mgrstat  56609621
 -2512> 2017-11-05 21:59:12.691380 7f2338baee00 10 mon.ceph03@-1(probing).mgrstat update_from_paxos v56609621 service_map e1
 -2511> 2017-11-05 21:59:12.691388 7f2338baee00 10 mon.ceph03@-1(probing).mgrstat check_subs
 -2510> 2017-11-05 21:59:12.691390 7f2338baee00 20 mon.ceph03@-1(probing).mgrstat update_logger
 -2509> 2017-11-05 21:59:12.691425 7f2338baee00 10 mon.ceph03@-1(probing).paxosservice(health 1..22) refresh
 -2508> 2017-11-05 21:59:12.691432 7f2338baee00 10 mon.ceph03@-1(probing).health update_from_paxos
 -2507> 2017-11-05 21:59:12.691463 7f2338baee00 20 mon.ceph03@-1(probing).health dump:{
    "quorum_health": {},
    "leader_health": {}
}

 -2506> 2017-11-05 21:59:12.691479 7f2338baee00 10 mon.ceph03@-1(probing).paxosservice(pgmap 56600067..56600783) post_refresh
 -2505> 2017-11-05 21:59:12.691481 7f2338baee00 10 mon.ceph03@-1(probing).paxosservice(mdsmap 1..1) post_refresh
 -2504> 2017-11-05 21:59:12.691483 7f2338baee00 10 mon.ceph03@-1(probing).paxosservice(osdmap 503..1105) post_refresh
 -2503> 2017-11-05 21:59:12.691485 7f2338baee00 10 mon.ceph03@-1(probing).paxosservice(logm 55596915..55597627) post_refresh
 -2502> 2017-11-05 21:59:12.691486 7f2338baee00 10 mon.ceph03@-1(probing).paxosservice(monmap 1..14) post_refresh
 -2501> 2017-11-05 21:59:12.691488 7f2338baee00 10 mon.ceph03@-1(probing).paxosservice(auth 26252..26331) post_refresh
 -2500> 2017-11-05 21:59:12.691490 7f2338baee00 10 mon.ceph03@-1(probing).paxosservice(mgr 1..394) post_refresh
 -2499> 2017-11-05 21:59:12.691491 7f2338baee00 10 mon.ceph03@-1(probing).paxosservice(mgrstat 56609513..56609621) post_refresh
 -2498> 2017-11-05 21:59:12.691493 7f2338baee00 10 mon.ceph03@-1(probing).paxosservice(health 1..22) post_refresh
 -2497> 2017-11-05 21:59:12.691528 7f2338baee00 10 mon.ceph03@-1(probing).health(0) init
 -2496> 2017-11-05 21:59:12.691685 7f2338baee00  2 auth: KeyRing::load: loaded key file /var/lib/ceph/mon/ceph-ceph03/keyring
 -2495> 2017-11-05 21:59:12.691696 7f2338baee00  5 asok(0x7f2342958380) register_command mon_status hook 0x7f23428e8890
 -2494> 2017-11-05 21:59:12.691703 7f2338baee00  5 asok(0x7f2342958380) register_command quorum_status hook 0x7f23428e8890
 -2493> 2017-11-05 21:59:12.691712 7f2338baee00  5 asok(0x7f2342958380) register_command sync_force hook 0x7f23428e8890
 -2492> 2017-11-05 21:59:12.691718 7f2338baee00  5 asok(0x7f2342958380) register_command add_bootstrap_peer_hint hook 0x7f23428e8890
 -2491> 2017-11-05 21:59:12.691725 7f2338baee00  5 asok(0x7f2342958380) register_command quorum enter hook 0x7f23428e8890
 -2490> 2017-11-05 21:59:12.691730 7f2338baee00  5 asok(0x7f2342958380) register_command quorum exit hook 0x7f23428e8890
 -2489> 2017-11-05 21:59:12.691737 7f2338baee00  5 asok(0x7f2342958380) register_command ops hook 0x7f23428e8890
 -2488> 2017-11-05 21:59:12.691742 7f2338baee00  5 asok(0x7f2342958380) register_command sessions hook 0x7f23428e8890
 -2487> 2017-11-05 21:59:12.691737 7f232d2de700  1 -- 10.10.1.23:0/0 learned_addr learned my addr 10.10.1.23:0/0
 -2486> 2017-11-05 21:59:12.691773 7f2338baee00  1 finished global_init_daemonize
 -2485> 2017-11-05 21:59:12.691788 7f2338baee00  1 -- 10.10.1.23:6789/0 start start
 -2484> 2017-11-05 21:59:12.691798 7f2338baee00  1 -- 10.10.1.23:0/0 start start
 -2483> 2017-11-05 21:59:12.691809 7f2338baee00  2 mon.ceph03@-1(probing) e14 init
 -2482> 2017-11-05 21:59:12.691941 7f232d2de700  0 -- 10.10.1.23:0/15195 >> 10.10.1.21:6806/22556 conn(0x7f2342dab800 :-1 s=STATE_CONNECTING_WAIT_CONNECT_REPLY_AUTH pgs=0 cs=0 l=0).handle_connect_reply connect got BADAUTHORIZER
 -2481> 2017-11-05 21:59:12.691972 7f2338baee00  1  Processor -- start
 -2480> 2017-11-05 21:59:12.692032 7f23322e8700 20 _process 0x7f2342d4ae00 1 [0,512)
 -2479> 2017-11-05 21:59:12.692034 7f2331ae7700 20 _process 0x7f2342d4ae00 2 [0,128)
 -2478> 2017-11-05 21:59:12.692076 7f23312e6700 20 _process 0x7f2342d4ae00 4 [0,128)
 -2477> 2017-11-05 21:59:12.692139 7f2338baee00  1  Processor -- start
 -2476> 2017-11-05 21:59:12.692193 7f2338baee00 10 mon.ceph03@-1(probing) e14 bootstrap
 -2475> 2017-11-05 21:59:12.692201 7f2338baee00 10 mon.ceph03@-1(probing) e14 sync_reset_requester
 -2474> 2017-11-05 21:59:12.692203 7f2338baee00 10 mon.ceph03@-1(probing) e14 unregister_cluster_logger - not registered
 -2473> 2017-11-05 21:59:12.692204 7f2338baee00 10 mon.ceph03@-1(probing) e14 cancel_probe_timeout (none scheduled)
 -2472> 2017-11-05 21:59:12.692208 7f2338baee00  0 mon.ceph03@-1(probing) e14  my rank is now 2 (was -1)
 -2471> 2017-11-05 21:59:12.692211 7f2338baee00  1 -- 10.10.1.23:6789/0 shutdown_connections
 -2470> 2017-11-05 21:59:12.692217 7f2338baee00 10 mon.ceph03@2(probing) e14 _reset
 -2469> 2017-11-05 21:59:12.692219 7f2338baee00 10 mon.ceph03@2(probing) e14 cancel_probe_timeout (none scheduled)
 -2468> 2017-11-05 21:59:12.692220 7f2338baee00 10 mon.ceph03@2(probing) e14 timecheck_finish
 -2467> 2017-11-05 21:59:12.692223 7f2338baee00 15 mon.ceph03@2(probing) e14 health_tick_stop
 -2466> 2017-11-05 21:59:12.692224 7f2338baee00 15 mon.ceph03@2(probing) e14 health_interval_stop
 -2465> 2017-11-05 21:59:12.692226 7f2338baee00 10 mon.ceph03@2(probing) e14 scrub_event_cancel
 -2464> 2017-11-05 21:59:12.692230 7f2338baee00 10 mon.ceph03@2(probing) e14 scrub_reset
 -2463> 2017-11-05 21:59:12.692232 7f2338baee00 10 mon.ceph03@2(probing).paxos(paxos recovering c 112907080..112907717) restart -- canceling timeouts
 -2462> 2017-11-05 21:59:12.692238 7f2338baee00 10 mon.ceph03@2(probing).paxosservice(pgmap 56600067..56600783) restart
 -2461> 2017-11-05 21:59:12.692241 7f2338baee00 10 mon.ceph03@2(probing).paxosservice(mdsmap 1..1) restart
 -2460> 2017-11-05 21:59:12.692243 7f2338baee00 10 mon.ceph03@2(probing).paxosservice(osdmap 503..1105) restart
 -2459> 2017-11-05 21:59:12.692245 7f2338baee00 10 mon.ceph03@2(probing).paxosservice(logm 55596915..55597627) restart
 -2458> 2017-11-05 21:59:12.692247 7f2338baee00 10 mon.ceph03@2(probing).paxosservice(monmap 1..14) restart
 -2457> 2017-11-05 21:59:12.692248 7f2338baee00 10 mon.ceph03@2(probing).paxosservice(auth 26252..26331) restart
 -2456> 2017-11-05 21:59:12.692250 7f2338baee00 10 mon.ceph03@2(probing).paxosservice(mgr 1..394) restart
 -2455> 2017-11-05 21:59:12.692252 7f2338baee00 10 mon.ceph03@2(probing).paxosservice(mgrstat 56609513..56609621) restart
 -2454> 2017-11-05 21:59:12.692253 7f2338baee00 10 mon.ceph03@2(probing).paxosservice(health 1..22) restart
 -2453> 2017-11-05 21:59:12.692257 7f2338baee00 10 mon.ceph03@2(probing) e14 cancel_probe_timeout (none scheduled)
 -2452> 2017-11-05 21:59:12.692260 7f2338baee00 10 mon.ceph03@2(probing) e14 reset_probe_timeout 0x7f2342ba0340 after 2 seconds
 -2451> 2017-11-05 21:59:12.692279 7f2338baee00 10 mon.ceph03@2(probing) e14 probing other monitors
 -2450> 2017-11-05 21:59:12.692282 7f2338baee00  1 -- 10.10.1.23:6789/0 _send_message--> mon.0 10.10.1.21:6789/0 -- mon_probe(probe 726a42b3-ae4c-4d0b-82c5-0a9065711b5f name ceph03) v6 -- ?+0 0x7f2342beadc0
 -2449> 2017-11-05 21:59:12.692325 7f2338baee00  1 -- 10.10.1.23:6789/0 --> 10.10.1.21:6789/0 -- mon_probe(probe 726a42b3-ae4c-4d0b-82c5-0a9065711b5f name ceph03) v6 -- 0x7f2342beadc0 con 0
 -2448> 2017-11-05 21:59:12.692341 7f2338baee00  1 -- 10.10.1.23:6789/0 _send_message--> mon.1 10.10.1.22:6789/0 -- mon_probe(probe 726a42b3-ae4c-4d0b-82c5-0a9065711b5f name ceph03) v6 -- ?+0 0x7f2342beab00
 -2447> 2017-11-05 21:59:12.692354 7f2338baee00  1 -- 10.10.1.23:6789/0 --> 10.10.1.22:6789/0 -- mon_probe(probe 726a42b3-ae4c-4d0b-82c5-0a9065711b5f name ceph03) v6 -- 0x7f2342beab00 con 0
 -2446> 2017-11-05 21:59:12.692522 7f232d2de700  0 -- 10.10.1.23:0/15195 >> 10.10.1.21:6806/22556 conn(0x7f2342dab800 :-1 s=STATE_CONNECTING_WAIT_CONNECT_REPLY_AUTH pgs=0 cs=0 l=0).handle_connect_reply connect got BADAUTHORIZER
 -2445> 2017-11-05 21:59:12.692820 7f232c2dc700 10 mon.ceph03@2(probing) e14 ms_get_authorizer for mon
 -2444> 2017-11-05 21:59:12.692931 7f232c2dc700 10 cephx: build_service_ticket service mon secret_id 18446744073709551615 ticket_info.ticket.name=mon.
 -2443> 2017-11-05 21:59:12.692929 7f232cadd700 10 mon.ceph03@2(probing) e14 ms_get_authorizer for mon
 -2442> 2017-11-05 21:59:12.693026 7f232cadd700 10 cephx: build_service_ticket service mon secret_id 18446744073709551615 ticket_info.ticket.name=mon.
 -2441> 2017-11-05 21:59:12.693346 7f232cadd700  5 -- 10.10.1.23:6789/0 >> 10.10.1.22:6789/0 conn(0x7f2342dd3000 :-1 s=STATE_CONNECTING_WAIT_CONNECT_REPLY_AUTH pgs=0 cs=0 l=0).handle_connect_reply connect got RETRY_GLOBAL 1539 chose new 1540
 -2440> 2017-11-05 21:59:12.693355 7f232c2dc700  5 -- 10.10.1.23:6789/0 >> 10.10.1.21:6789/0 conn(0x7f2342dd4800 :-1 s=STATE_CONNECTING_WAIT_CONNECT_REPLY_AUTH pgs=0 cs=0 l=0).handle_connect_reply connect got RETRY_GLOBAL 1538 chose new 1541
 -2439> 2017-11-05 21:59:12.693394 7f232cadd700 10 mon.ceph03@2(probing) e14 ms_get_authorizer for mon
 -2438> 2017-11-05 21:59:12.693418 7f232c2dc700 10 mon.ceph03@2(probing) e14 ms_get_authorizer for mon
 -2437> 2017-11-05 21:59:12.693504 7f232c2dc700 10 cephx: build_service_ticket service mon secret_id 18446744073709551615 ticket_info.ticket.name=mon.
 -2436> 2017-11-05 21:59:12.693751 7f232cadd700 10 cephx: build_service_ticket service mon secret_id 18446744073709551615 ticket_info.ticket.name=mon.
 -2435> 2017-11-05 21:59:12.694011 7f23322e8700 10 osdmap epoch 1105 mapping took 0.004593 seconds
 -2434> 2017-11-05 21:59:12.694022 7f23322e8700 10 mon.ceph03@2(probing).osd e1105 update_creating_pgs 0 pgs creating, 0 pools in queue
 -2433> 2017-11-05 21:59:12.694111 7f232c2dc700  5 -- 10.10.1.23:6789/0 >> 10.10.1.21:6789/0 conn(0x7f2342dd4800 :-1 s=STATE_CONNECTING_WAIT_CONNECT_REPLY_AUTH pgs=0 cs=0 l=0).handle_connect_reply connect got RETRY_GLOBAL 1538 chose new 1542
 -2432> 2017-11-05 21:59:12.694142 7f232c2dc700 10 mon.ceph03@2(probing) e14 ms_get_authorizer for mon
 -2431> 2017-11-05 21:59:12.694162 7f232cadd700  5 -- 10.10.1.23:6789/0 >> 10.10.1.22:6789/0 conn(0x7f2342dd3000 :-1 s=STATE_CONNECTING_WAIT_CONNECT_REPLY_AUTH pgs=0 cs=0 l=0).handle_connect_reply connect got RETRY_GLOBAL 1539 chose new 1543
 -2430> 2017-11-05 21:59:12.694190 7f232cadd700 10 mon.ceph03@2(probing) e14 ms_get_authorizer for mon
 -2429> 2017-11-05 21:59:12.694201 7f232c2dc700 10 cephx: build_service_ticket service mon secret_id 18446744073709551615 ticket_info.ticket.name=mon.
 -2428> 2017-11-05 21:59:12.694246 7f232cadd700 10 cephx: build_service_ticket service mon secret_id 18446744073709551615 ticket_info.ticket.name=mon.
 -2427> 2017-11-05 21:59:12.694485 7f232c2dc700 10 In get_auth_session_handler for protocol 2
 -2426> 2017-11-05 21:59:12.694490 7f232cadd700 10 In get_auth_session_handler for protocol 2
 -2425> 2017-11-05 21:59:12.694558 7f232c2dc700 10 _calc_signature seq 1 front_crc_ = 637078008 middle_crc = 0 data_crc = 0 sig = 8409844418224713746
 -2424> 2017-11-05 21:59:12.694569 7f232c2dc700 20 Putting signature in client message(seq # 1): sig = 8409844418224713746
 -2423> 2017-11-05 21:59:12.694578 7f232cadd700 10 _calc_signature seq 1 front_crc_ = 637078008 middle_crc = 0 data_crc = 0 sig = 16785736216537150546
 -2422> 2017-11-05 21:59:12.694584 7f232cadd700 20 Putting signature in client message(seq # 1): sig = 16785736216537150546
 -2421> 2017-11-05 21:59:12.694882 7f232c2dc700 10 _calc_signature seq 248515223 front_crc_ = 750666649 middle_crc = 0 data_crc = 0 sig = 5792814148247093935
 -2420> 2017-11-05 21:59:12.694892 7f232c2dc700  0 -- 10.10.1.23:6789/0 >> 10.10.1.21:6789/0 conn(0x7f2342dd4800 :-1 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=1124301 cs=1 l=0).process missed message?  skipped from seq 0 to 248515223
 -2419> 2017-11-05 21:59:12.694909 7f232c2dc700  5 -- 10.10.1.23:6789/0 >> 10.10.1.21:6789/0 conn(0x7f2342dd4800 :-1 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=1124301 cs=1 l=0). rx mon.0 seq 248515223 0x7f2342beadc0 mon_probe(reply 726a42b3-ae4c-4d0b-82c5-0a9065711b5f name ceph01mon quorum 0,1 paxos( fc 112907080 lc 112907738 )) v6
 -2418> 2017-11-05 21:59:12.694960 7f23302e4700  1 -- 10.10.1.23:6789/0 <== mon.0 10.10.1.21:6789/0 248515223 ==== mon_probe(reply 726a42b3-ae4c-4d0b-82c5-0a9065711b5f name ceph01mon quorum 0,1 paxos( fc 112907080 lc 112907738 )) v6 ==== 494+0+0 (750666649 0 0) 0x7f2342beadc0 con 0x7f2342dd4800
 -2417> 2017-11-05 21:59:12.694997 7f23302e4700 10 mon.ceph03@2(probing) e14 _ms_dispatch new session 0x7f2342930900 MonSession(mon.0 10.10.1.21:6789/0 is open ) features 0x1ffddff8eea4fffb
 -2416> 2017-11-05 21:59:12.695008 7f23302e4700  5 mon.ceph03@2(probing) e14 _ms_dispatch setting monitor caps on this connection
 -2415> 2017-11-05 21:59:12.695032 7f23302e4700 20 mon.ceph03@2(probing) e14  caps allow *
 -2414> 2017-11-05 21:59:12.695040 7f23302e4700 20 is_capable service=mon command= read on cap allow *
 -2413> 2017-11-05 21:59:12.695043 7f23302e4700 20  allow so far , doing grant allow *
 -2412> 2017-11-05 21:59:12.695045 7f23302e4700 20  allow all
 -2411> 2017-11-05 21:59:12.695047 7f23302e4700 10 mon.ceph03@2(probing) e14 handle_probe mon_probe(reply 726a42b3-ae4c-4d0b-82c5-0a9065711b5f name ceph01mon quorum 0,1 paxos( fc 112907080 lc 112907738 )) v6
 -2410> 2017-11-05 21:59:12.695052 7f23302e4700 10 mon.ceph03@2(probing) e14 handle_probe_reply mon.0 10.10.1.21:6789/0mon_probe(reply 726a42b3-ae4c-4d0b-82c5-0a9065711b5f name ceph01mon quorum 0,1 paxos( fc 112907080 lc 112907738 )) v6
 -2409> 2017-11-05 21:59:12.695059 7f23302e4700 10 mon.ceph03@2(probing) e14  monmap is e14: 3 mons at {ceph01mon=10.10.1.21:6789/0,ceph02mon=10.10.1.22:6789/0,ceph03=10.10.1.23:6789/0}
 -2408> 2017-11-05 21:59:12.695100 7f23302e4700 10 mon.ceph03@2(probing) e14  peer name is ceph01mon
 -2407> 2017-11-05 21:59:12.695107 7f23302e4700 10 mon.ceph03@2(probing) e14  peer paxos last version 112907738 vs my version 112907717 (too far ahead)
 -2406> 2017-11-05 21:59:12.695111 7f23302e4700 10 mon.ceph03@2(probing) e14 cancel_probe_timeout 0x7f2342ba0340
 -2405> 2017-11-05 21:59:12.695115 7f23302e4700 10 mon.ceph03@2(probing) e14 sync_start mon.0 10.10.1.21:6789/0 recent
 -2404> 2017-11-05 21:59:12.695122 7f23302e4700 10 mon.ceph03@2(synchronizing) e14 sync_reset_provider
 -2403> 2017-11-05 21:59:12.695125 7f23302e4700 10 mon.ceph03@2(synchronizing) e14 sync_reset_timeout
 -2402> 2017-11-05 21:59:12.695130 7f23302e4700  1 -- 10.10.1.23:6789/0 _send_message--> mon.0 10.10.1.21:6789/0 -- mon_sync(get_cookie_recent lc 112907717) v2 -- ?+0 0x7f2342beb080
 -2401> 2017-11-05 21:59:12.695138 7f23302e4700  1 -- 10.10.1.23:6789/0 --> 10.10.1.21:6789/0 -- mon_sync(get_cookie_recent lc 112907717) v2 -- 0x7f2342beb080 con 0
 -2400> 2017-11-05 21:59:12.695183 7f232c2dc700 10 _calc_signature seq 2 front_crc_ = 2718256703 middle_crc = 0 data_crc = 0 sig = 4541292504390853050
 -2399> 2017-11-05 21:59:12.695193 7f232c2dc700 20 Putting signature in client message(seq # 2): sig = 4541292504390853050
 -2398> 2017-11-05 21:59:12.695462 7f232c2dc700 10 _calc_signature seq 248515224 front_crc_ = 2747452703 middle_crc = 0 data_crc = 0 sig = 10051698588873077381
 -2397> 2017-11-05 21:59:12.695472 7f232c2dc700  5 -- 10.10.1.23:6789/0 >> 10.10.1.21:6789/0 conn(0x7f2342dd4800 :-1 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=1124301 cs=1 l=0). rx mon.0 seq 248515224 0x7f2342beb080 mon_sync(cookie cookie 5737807875 lc 112907717) v2
 -2396> 2017-11-05 21:59:12.695505 7f23302e4700  1 -- 10.10.1.23:6789/0 <== mon.0 10.10.1.21:6789/0 248515224 ==== mon_sync(cookie cookie 5737807875 lc 112907717) v2 ==== 88+0+0 (2747452703 0 0) 0x7f2342beb080 con 0x7f2342dd4800
 -2395> 2017-11-05 21:59:12.695521 7f23302e4700 20 mon.ceph03@2(synchronizing) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
 -2394> 2017-11-05 21:59:12.695529 7f23302e4700 20 mon.ceph03@2(synchronizing) e14  caps allow *
 -2393> 2017-11-05 21:59:12.695532 7f23302e4700 20 is_capable service=mon command= read on cap allow *
 -2392> 2017-11-05 21:59:12.695534 7f23302e4700 20  allow so far , doing grant allow *
 -2391> 2017-11-05 21:59:12.695535 7f23302e4700 20  allow all
 -2390> 2017-11-05 21:59:12.695537 7f23302e4700 10 mon.ceph03@2(synchronizing) e14 handle_sync mon_sync(cookie cookie 5737807875 lc 112907717) v2
 -2389> 2017-11-05 21:59:12.695540 7f23302e4700 10 mon.ceph03@2(synchronizing) e14 handle_sync_cookie mon_sync(cookie cookie 5737807875 lc 112907717) v2
 -2388> 2017-11-05 21:59:12.695555 7f23302e4700 10 mon.ceph03@2(synchronizing) e14 sync_reset_timeout
 -2387> 2017-11-05 21:59:12.695559 7f23302e4700 20 mon.ceph03@2(synchronizing) e14 sync_get_next_chunk cookie 5737807875 provider mon.0 10.10.1.21:6789/0
 -2386> 2017-11-05 21:59:12.695565 7f23302e4700  1 -- 10.10.1.23:6789/0 _send_message--> mon.0 10.10.1.21:6789/0 -- mon_sync(get_chunk cookie 5737807875) v2 -- ?+0 0x7f2342beadc0
 -2385> 2017-11-05 21:59:12.695571 7f23302e4700  1 -- 10.10.1.23:6789/0 --> 10.10.1.21:6789/0 -- mon_sync(get_chunk cookie 5737807875) v2 -- 0x7f2342beadc0 con 0
 -2384> 2017-11-05 21:59:12.695612 7f232c2dc700 10 _calc_signature seq 3 front_crc_ = 2405319517 middle_crc = 0 data_crc = 0 sig = 7150424104754382822
 -2383> 2017-11-05 21:59:12.695621 7f232c2dc700 20 Putting signature in client message(seq # 3): sig = 7150424104754382822
 -2382> 2017-11-05 21:59:12.697237 7f232c2dc700 10 _calc_signature seq 248515225 front_crc_ = 711971838 middle_crc = 0 data_crc = 0 sig = 17556540721934124985
 -2381> 2017-11-05 21:59:12.697248 7f232c2dc700  5 -- 10.10.1.23:6789/0 >> 10.10.1.21:6789/0 conn(0x7f2342dd4800 :-1 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=1124301 cs=1 l=0). rx mon.0 seq 248515225 0x7f2342beadc0 mon_sync(last_chunk cookie 5737807875 lc 112907738 bl 112078 bytes) v2
 -2380> 2017-11-05 21:59:12.697305 7f23302e4700  1 -- 10.10.1.23:6789/0 <== mon.0 10.10.1.21:6789/0 248515225 ==== mon_sync(last_chunk cookie 5737807875 lc 112907738 bl 112078 bytes) v2 ==== 112166+0+0 (711971838 0 0) 0x7f2342beadc0 con 0x7f2342dd4800
 -2379> 2017-11-05 21:59:12.697338 7f23302e4700 20 mon.ceph03@2(synchronizing) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
 -2378> 2017-11-05 21:59:12.697348 7f23302e4700 20 mon.ceph03@2(synchronizing) e14  caps allow *
 -2377> 2017-11-05 21:59:12.697351 7f23302e4700 20 is_capable service=mon command= read on cap allow *
 -2376> 2017-11-05 21:59:12.697354 7f23302e4700 20  allow so far , doing grant allow *
 -2375> 2017-11-05 21:59:12.697355 7f23302e4700 20  allow all
 -2374> 2017-11-05 21:59:12.697357 7f23302e4700 10 mon.ceph03@2(synchronizing) e14 handle_sync mon_sync(last_chunk cookie 5737807875 lc 112907738 bl 112078 bytes) v2
 -2373> 2017-11-05 21:59:12.697362 7f23302e4700 10 mon.ceph03@2(synchronizing) e14 handle_sync_chunk mon_sync(last_chunk cookie 5737807875 lc 112907738 bl 112078 bytes) v2
 -2372> 2017-11-05 21:59:12.701065 7f232cadd700 10 _calc_signature seq 123007630 front_crc_ = 2407865230 middle_crc = 0 data_crc = 0 sig = 13627423979967733608
 -2371> 2017-11-05 21:59:12.701085 7f232cadd700  0 -- 10.10.1.23:6789/0 >> 10.10.1.22:6789/0 conn(0x7f2342dd3000 :-1 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=973717 cs=1 l=0).process missed message?  skipped from seq 0 to 123007630
 -2370> 2017-11-05 21:59:12.701103 7f232cadd700  5 -- 10.10.1.23:6789/0 >> 10.10.1.22:6789/0 conn(0x7f2342dd3000 :-1 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=973717 cs=1 l=0). rx mon.1 seq 123007630 0x7f2342beab00 mon_probe(reply 726a42b3-ae4c-4d0b-82c5-0a9065711b5f name ceph02mon quorum 0,1 paxos( fc 112907080 lc 112907738 )) v6
 -2369> 2017-11-05 21:59:12.751656 7f23302e4700 10 mon.ceph03@2(synchronizing) e14 handle_sync_chunk applying recent paxos transactions as we go
 -2368> 2017-11-05 21:59:12.751676 7f23302e4700 10 mon.ceph03@2(synchronizing).paxos(paxos recovering c 112907080..112907717) read_and_prepare_transactions first 112907718 last 112907738
 -2367> 2017-11-05 21:59:12.751915 7f23302e4700 15 mon.ceph03@2(synchronizing).paxos(paxos recovering c 112907080..112907717) read_and_prepare_transactions total versions 20
 -2366> 2017-11-05 21:59:12.796201 7f23302e4700 10 mon.ceph03@2(synchronizing).paxos(paxos recovering c 112907080..112907738) init last_pn: 12300 accepted_pn: 13500 last_committed: 112907738 first_committed: 112907080
 -2365> 2017-11-05 21:59:12.796218 7f23302e4700 10 mon.ceph03@2(synchronizing).paxos(paxos recovering c 112907080..112907738) init
 -2364> 2017-11-05 21:59:12.796243 7f23302e4700 10 mon.ceph03@2(synchronizing) e14 sync_finish lc 112907738 from mon.0 10.10.1.21:6789/0
 -2363> 2017-11-05 21:59:12.840316 7f23302e4700 10 mon.ceph03@2(synchronizing) e14 init_paxos
 -2362> 2017-11-05 21:59:12.840411 7f23302e4700 10 mon.ceph03@2(synchronizing).paxos(paxos recovering c 112907080..112907738) init last_pn: 12300 accepted_pn: 13500 last_committed: 112907738 first_committed: 112907080
 -2361> 2017-11-05 21:59:12.840421 7f23302e4700 10 mon.ceph03@2(synchronizing).paxos(paxos recovering c 112907080..112907738) init
 -2360> 2017-11-05 21:59:12.840439 7f23302e4700  1 mon.ceph03@2(synchronizing).mds e1 Unable to load 'last_metadata'
 -2359> 2017-11-05 21:59:12.840477 7f23302e4700 10 mon.ceph03@2(synchronizing).health init
 -2358> 2017-11-05 21:59:12.840482 7f23302e4700 10 mon.ceph03@2(synchronizing) e14 refresh_from_paxos
 -2357> 2017-11-05 21:59:12.840496 7f23302e4700 10 mon.ceph03@2(synchronizing) e14 refresh_from_paxos no cluster_fingerprint
 -2356> 2017-11-05 21:59:12.840526 7f23302e4700 10 mon.ceph03@2(synchronizing).paxosservice(pgmap 56600067..56600783) refresh
 -2355> 2017-11-05 21:59:12.840563 7f23302e4700 10 mon.ceph03@2(synchronizing).paxosservice(mdsmap 1..1) refresh
 -2354> 2017-11-05 21:59:12.840602 7f23302e4700 10 mon.ceph03@2(synchronizing).paxosservice(osdmap 503..1105) refresh
 -2353> 2017-11-05 21:59:12.840629 7f23302e4700 10 mon.ceph03@2(synchronizing).paxosservice(logm 55596915..55597634) refresh
 -2352> 2017-11-05 21:59:12.840634 7f23302e4700 10 mon.ceph03@2(synchronizing).log v55597634 update_from_paxos
 -2351> 2017-11-05 21:59:12.840644 7f23302e4700 10 mon.ceph03@2(synchronizing).log v55597634 update_from_paxos version 55597634 summary v 55597627
 -2350> 2017-11-05 21:59:12.840655 7f23302e4700 10 mon.ceph03@2(synchronizing).log v55597634 update_from_paxos latest full 55597633
 -2349> 2017-11-05 21:59:12.840669 7f23302e4700  7 mon.ceph03@2(synchronizing).log v55597634 update_from_paxos loading summary e55597633
 -2348> 2017-11-05 21:59:12.840809 7f23302e4700  7 mon.ceph03@2(synchronizing).log v55597634 update_from_paxos loaded summary e55597633
 -2347> 2017-11-05 21:59:12.840824 7f23302e4700  7 mon.ceph03@2(synchronizing).log v55597634 update_from_paxos applying incremental log 55597634 2017-11-05 21:59:10.716523 mgr.ceph01mgr client.54805773 10.10.1.21:0/1619113657 1817 : audit [DBG] from='client.55026393 10.10.1.21:0/2834401706' entity='client.admin' cmd=[{"prefix": "osd pool stats", "target": ["mgr", ""], "format": "json"}]: dispatch
 -2346> 2017-11-05 21:59:12.840844 7f23302e4700 20 mon.ceph03@2(synchronizing).log v55597634 update_from_paxos logging for channel 'audit' to file '/var/log/ceph/ceph.audit.log'
 -2345> 2017-11-05 21:59:12.840853 7f23302e4700 15 mon.ceph03@2(synchronizing).log v55597634 update_from_paxos logging for 0 channels
 -2344> 2017-11-05 21:59:12.840858 7f23302e4700 10 mon.ceph03@2(synchronizing).log v55597634 check_subs
 -2343> 2017-11-05 21:59:12.840895 7f23302e4700 10 mon.ceph03@2(synchronizing).paxosservice(monmap 1..14) refresh
 -2342> 2017-11-05 21:59:12.840921 7f23302e4700 10 mon.ceph03@2(synchronizing).paxosservice(auth 26252..26331) refresh
 -2341> 2017-11-05 21:59:12.840926 7f23302e4700 10 mon.ceph03@2(synchronizing).auth v26331 update_from_paxos
 -2340> 2017-11-05 21:59:12.840960 7f23302e4700 10 mon.ceph03@2(synchronizing).paxosservice(mgr 1..394) refresh
 -2339> 2017-11-05 21:59:12.840971 7f23302e4700  4 mgrc handle_mgr_map Got map version 394
 -2338> 2017-11-05 21:59:12.840974 7f23302e4700  4 mgrc handle_mgr_map Active mgr is now 10.10.1.21:6806/22556
 -2337> 2017-11-05 21:59:12.841022 7f23302e4700 10 mon.ceph03@2(synchronizing).paxosservice(mgrstat 56609513..56609633) refresh
 -2336> 2017-11-05 21:59:12.841027 7f23302e4700 10 mon.ceph03@2(synchronizing).mgrstat  56609633
 -2335> 2017-11-05 21:59:12.841063 7f23302e4700 10 mon.ceph03@2(synchronizing).mgrstat update_from_paxos v56609633 service_map e1
 -2334> 2017-11-05 21:59:12.841068 7f23302e4700 10 mon.ceph03@2(synchronizing).mgrstat check_subs
 -2333> 2017-11-05 21:59:12.841070 7f23302e4700 20 mon.ceph03@2(synchronizing).mgrstat update_logger
 -2332> 2017-11-05 21:59:12.841097 7f23302e4700 10 mon.ceph03@2(synchronizing).paxosservice(health 1..23) refresh
 -2331> 2017-11-05 21:59:12.841104 7f23302e4700 10 mon.ceph03@2(synchronizing).health update_from_paxos
 -2330> 2017-11-05 21:59:12.841121 7f23302e4700 20 mon.ceph03@2(synchronizing).health dump:{
    "quorum_health": {},
    "leader_health": {
        "MON_DOWN": {
            "severity": "HEALTH_WARN",
            "summary": {
                "message": "1/3 mons down, quorum ceph01mon,ceph02mon"
            },
            "detail": [
                {
                    "message": "mon.ceph03 (rank 2) addr 10.10.1.23:6789/0 is down (out of quorum)"
                }
            ]
        }
    }
}

 -2329> 2017-11-05 21:59:12.841151 7f23302e4700 10 mon.ceph03@2(synchronizing).paxosservice(pgmap 56600067..56600783) post_refresh
 -2328> 2017-11-05 21:59:12.841154 7f23302e4700 10 mon.ceph03@2(synchronizing).paxosservice(mdsmap 1..1) post_refresh
 -2327> 2017-11-05 21:59:12.841155 7f23302e4700 10 mon.ceph03@2(synchronizing).paxosservice(osdmap 503..1105) post_refresh
 -2326> 2017-11-05 21:59:12.841156 7f23302e4700 10 mon.ceph03@2(synchronizing).paxosservice(logm 55596915..55597634) post_refresh
 -2325> 2017-11-05 21:59:12.841158 7f23302e4700 10 mon.ceph03@2(synchronizing).paxosservice(monmap 1..14) post_refresh
 -2324> 2017-11-05 21:59:12.841159 7f23302e4700 10 mon.ceph03@2(synchronizing).paxosservice(auth 26252..26331) post_refresh
 -2323> 2017-11-05 21:59:12.841162 7f23302e4700 10 mon.ceph03@2(synchronizing).paxosservice(mgr 1..394) post_refresh
 -2322> 2017-11-05 21:59:12.841163 7f23302e4700 10 mon.ceph03@2(synchronizing).paxosservice(mgrstat 56609513..56609633) post_refresh
 -2321> 2017-11-05 21:59:12.841165 7f23302e4700 10 mon.ceph03@2(synchronizing).paxosservice(health 1..23) post_refresh
 -2320> 2017-11-05 21:59:12.841212 7f23302e4700 10 mon.ceph03@2(synchronizing) e14 bootstrap
 -2319> 2017-11-05 21:59:12.841215 7f23302e4700 10 mon.ceph03@2(synchronizing) e14 sync_reset_requester
 -2318> 2017-11-05 21:59:12.841218 7f23302e4700 10 mon.ceph03@2(synchronizing) e14 unregister_cluster_logger - not registered
 -2317> 2017-11-05 21:59:12.841219 7f23302e4700 10 mon.ceph03@2(synchronizing) e14 cancel_probe_timeout (none scheduled)
 -2316> 2017-11-05 21:59:12.841222 7f23302e4700 10 mon.ceph03@2(probing) e14 _reset
 -2315> 2017-11-05 21:59:12.841223 7f23302e4700 10 mon.ceph03@2(probing) e14 cancel_probe_timeout (none scheduled)
 -2314> 2017-11-05 21:59:12.841224 7f23302e4700 10 mon.ceph03@2(probing) e14 timecheck_finish
 -2313> 2017-11-05 21:59:12.841226 7f23302e4700 15 mon.ceph03@2(probing) e14 health_tick_stop
 -2312> 2017-11-05 21:59:12.841227 7f23302e4700 15 mon.ceph03@2(probing) e14 health_interval_stop
 -2311> 2017-11-05 21:59:12.841228 7f23302e4700 10 mon.ceph03@2(probing) e14 scrub_event_cancel
 -2310> 2017-11-05 21:59:12.841230 7f23302e4700 10 mon.ceph03@2(probing) e14 scrub_reset
 -2309> 2017-11-05 21:59:12.841232 7f23302e4700 10 mon.ceph03@2(probing).paxos(paxos recovering c 112907080..112907738) restart -- canceling timeouts
 -2308> 2017-11-05 21:59:12.841235 7f23302e4700 10 mon.ceph03@2(probing).paxosservice(pgmap 56600067..56600783) restart
 -2307> 2017-11-05 21:59:12.841237 7f23302e4700 10 mon.ceph03@2(probing).paxosservice(mdsmap 1..1) restart
 -2306> 2017-11-05 21:59:12.841239 7f23302e4700 10 mon.ceph03@2(probing).paxosservice(osdmap 503..1105) restart
 -2305> 2017-11-05 21:59:12.841242 7f23302e4700 10 mon.ceph03@2(probing).paxosservice(logm 55596915..55597634) restart
 -2304> 2017-11-05 21:59:12.841243 7f23302e4700 10 mon.ceph03@2(probing).paxosservice(monmap 1..14) restart
 -2303> 2017-11-05 21:59:12.841244 7f23302e4700 10 mon.ceph03@2(probing).paxosservice(auth 26252..26331) restart
 -2302> 2017-11-05 21:59:12.841246 7f23302e4700 10 mon.ceph03@2(probing).paxosservice(mgr 1..394) restart
 -2301> 2017-11-05 21:59:12.841247 7f23302e4700 10 mon.ceph03@2(probing).paxosservice(mgrstat 56609513..56609633) restart
 -2300> 2017-11-05 21:59:12.841250 7f23302e4700 10 mon.ceph03@2(probing).paxosservice(health 1..23) restart
 -2299> 2017-11-05 21:59:12.841253 7f23302e4700 10 mon.ceph03@2(probing) e14 cancel_probe_timeout (none scheduled)
 -2298> 2017-11-05 21:59:12.841260 7f23302e4700 10 mon.ceph03@2(probing) e14 reset_probe_timeout 0x7f2342ba02b0 after 2 seconds
 -2297> 2017-11-05 21:59:12.841269 7f23302e4700 10 mon.ceph03@2(probing) e14 probing other monitors
 -2296> 2017-11-05 21:59:12.841272 7f23302e4700  1 -- 10.10.1.23:6789/0 _send_message--> mon.0 10.10.1.21:6789/0 -- mon_probe(probe 726a42b3-ae4c-4d0b-82c5-0a9065711b5f name ceph03) v6 -- ?+0 0x7f2342beb080
 -2295> 2017-11-05 21:59:12.841283 7f23302e4700  1 -- 10.10.1.23:6789/0 --> 10.10.1.21:6789/0 -- mon_probe(probe 726a42b3-ae4c-4d0b-82c5-0a9065711b5f name ceph03) v6 -- 0x7f2342beb080 con 0
 -2294> 2017-11-05 21:59:12.841301 7f23302e4700  1 -- 10.10.1.23:6789/0 _send_message--> mon.1 10.10.1.22:6789/0 -- mon_probe(probe 726a42b3-ae4c-4d0b-82c5-0a9065711b5f name ceph03) v6 -- ?+0 0x7f2342bebb80
 -2293> 2017-11-05 21:59:12.841307 7f23302e4700  1 -- 10.10.1.23:6789/0 --> 10.10.1.22:6789/0 -- mon_probe(probe 726a42b3-ae4c-4d0b-82c5-0a9065711b5f name ceph03) v6 -- 0x7f2342bebb80 con 0
 -2292> 2017-11-05 21:59:12.841350 7f23302e4700  1 -- 10.10.1.23:6789/0 <== mon.1 10.10.1.22:6789/0 123007630 ==== mon_probe(reply 726a42b3-ae4c-4d0b-82c5-0a9065711b5f name ceph02mon quorum 0,1 paxos( fc 112907080 lc 112907738 )) v6 ==== 494+0+0 (2407865230 0 0) 0x7f2342beab00 con 0x7f2342dd3000
 -2291> 2017-11-05 21:59:12.841369 7f23302e4700 10 mon.ceph03@2(probing) e14 _ms_dispatch new session 0x7f2342930c00 MonSession(mon.1 10.10.1.22:6789/0 is open ) features 0x1ffddff8eea4fffb
 -2290> 2017-11-05 21:59:12.841378 7f23302e4700  5 mon.ceph03@2(probing) e14 _ms_dispatch setting monitor caps on this connection
 -2289> 2017-11-05 21:59:12.841381 7f23302e4700 20 mon.ceph03@2(probing) e14  caps allow *
 -2288> 2017-11-05 21:59:12.841376 7f232c2dc700 10 _calc_signature seq 4 front_crc_ = 637078008 middle_crc = 0 data_crc = 0 sig = 486809900323904982
 -2287> 2017-11-05 21:59:12.841385 7f23302e4700 20 is_capable service=mon command= read on cap allow *
 -2286> 2017-11-05 21:59:12.841381 7f232cadd700 10 _calc_signature seq 2 front_crc_ = 637078008 middle_crc = 0 data_crc = 0 sig = 113191317311269439
 -2285> 2017-11-05 21:59:12.841387 7f23302e4700 20  allow so far , doing grant allow *
 -2284> 2017-11-05 21:59:12.841388 7f23302e4700 20  allow all
 -2283> 2017-11-05 21:59:12.841388 7f232c2dc700 20 Putting signature in client message(seq # 4): sig = 486809900323904982
 -2282> 2017-11-05 21:59:12.841389 7f232cadd700 20 Putting signature in client message(seq # 2): sig = 113191317311269439
 -2281> 2017-11-05 21:59:12.841393 7f23302e4700 10 mon.ceph03@2(probing) e14 handle_probe mon_probe(reply 726a42b3-ae4c-4d0b-82c5-0a9065711b5f name ceph02mon quorum 0,1 paxos( fc 112907080 lc 112907738 )) v6
 -2280> 2017-11-05 21:59:12.841397 7f23302e4700 10 mon.ceph03@2(probing) e14 handle_probe_reply mon.1 10.10.1.22:6789/0mon_probe(reply 726a42b3-ae4c-4d0b-82c5-0a9065711b5f name ceph02mon quorum 0,1 paxos( fc 112907080 lc 112907738 )) v6
 -2279> 2017-11-05 21:59:12.841401 7f23302e4700 10 mon.ceph03@2(probing) e14  monmap is e14: 3 mons at {ceph01mon=10.10.1.21:6789/0,ceph02mon=10.10.1.22:6789/0,ceph03=10.10.1.23:6789/0}
 -2278> 2017-11-05 21:59:12.841420 7f23302e4700 10 mon.ceph03@2(probing) e14  peer name is ceph02mon
 -2277> 2017-11-05 21:59:12.841422 7f23302e4700 10 mon.ceph03@2(probing) e14  existing quorum 0,1
 -2276> 2017-11-05 21:59:12.841424 7f23302e4700 10 mon.ceph03@2(probing) e14  peer paxos version 112907738 vs my version 112907738 (ok)
 -2275> 2017-11-05 21:59:12.841427 7f23302e4700 10 mon.ceph03@2(probing) e14 start_election
 -2274> 2017-11-05 21:59:12.841428 7f23302e4700 10 mon.ceph03@2(probing) e14 _reset
 -2273> 2017-11-05 21:59:12.841429 7f23302e4700 10 mon.ceph03@2(probing) e14 cancel_probe_timeout 0x7f2342ba02b0
 -2272> 2017-11-05 21:59:12.841431 7f23302e4700 10 mon.ceph03@2(probing) e14 timecheck_finish
 -2271> 2017-11-05 21:59:12.841433 7f23302e4700 15 mon.ceph03@2(probing) e14 health_tick_stop
 -2270> 2017-11-05 21:59:12.841434 7f23302e4700 15 mon.ceph03@2(probing) e14 health_interval_stop
 -2269> 2017-11-05 21:59:12.841435 7f23302e4700 10 mon.ceph03@2(probing) e14 scrub_event_cancel
 -2268> 2017-11-05 21:59:12.841436 7f23302e4700 10 mon.ceph03@2(probing) e14 scrub_reset
 -2267> 2017-11-05 21:59:12.841440 7f23302e4700 10 mon.ceph03@2(probing).paxos(paxos recovering c 112907080..112907738) restart -- canceling timeouts
 -2266> 2017-11-05 21:59:12.841443 7f23302e4700 10 mon.ceph03@2(probing).paxosservice(pgmap 56600067..56600783) restart
 -2265> 2017-11-05 21:59:12.841445 7f23302e4700 10 mon.ceph03@2(probing).paxosservice(mdsmap 1..1) restart
 -2264> 2017-11-05 21:59:12.841446 7f23302e4700 10 mon.ceph03@2(probing).paxosservice(osdmap 503..1105) restart
 -2263> 2017-11-05 21:59:12.841448 7f23302e4700 10 mon.ceph03@2(probing).paxosservice(logm 55596915..55597634) restart
 -2262> 2017-11-05 21:59:12.841449 7f23302e4700 10 mon.ceph03@2(probing).paxosservice(monmap 1..14) restart
 -2261> 2017-11-05 21:59:12.841450 7f23302e4700 10 mon.ceph03@2(probing).paxosservice(auth 26252..26331) restart
 -2260> 2017-11-05 21:59:12.841452 7f23302e4700 10 mon.ceph03@2(probing).paxosservice(mgr 1..394) restart
 -2259> 2017-11-05 21:59:12.841453 7f23302e4700 10 mon.ceph03@2(probing).paxosservice(mgrstat 56609513..56609633) restart
 -2258> 2017-11-05 21:59:12.841454 7f23302e4700 10 mon.ceph03@2(probing).paxosservice(health 1..23) restart
 -2257> 2017-11-05 21:59:12.841466 7f23302e4700  0 log_channel(cluster) log [INF] : mon.ceph03 calling new monitor election
 -2256> 2017-11-05 21:59:12.841469 7f23302e4700 10 log_client _send_to_monlog to self
 -2255> 2017-11-05 21:59:12.841470 7f23302e4700 10 log_client  log_queue is 1 last_log 1 sent 0 num 1 unsent 1 sending 1
 -2254> 2017-11-05 21:59:12.841473 7f23302e4700 10 log_client  will send 2017-11-05 21:59:12.841468 mon.ceph03 mon.2 10.10.1.23:6789/0 1 : cluster [INF] mon.ceph03 calling new monitor election
 -2253> 2017-11-05 21:59:12.841482 7f23302e4700  1 -- 10.10.1.23:6789/0 --> 10.10.1.23:6789/0 -- log(1 entries from seq 1 at 2017-11-05 21:59:12.841468) v1 -- 0x7f234296e300 con 0
 -2252> 2017-11-05 21:59:12.841494 7f23302e4700  5 mon.ceph03@2(electing).elector(0) start -- can i be leader?
 -2251> 2017-11-05 21:59:12.841515 7f23302e4700  1 mon.ceph03@2(electing).elector(338) init, last seen epoch 338
 -2250> 2017-11-05 21:59:12.841517 7f23302e4700 10 mon.ceph03@2(electing).elector(338) bump_epoch 338 to 339
 -2249> 2017-11-05 21:59:12.841793 7f232c2dc700 10 _calc_signature seq 248515226 front_crc_ = 750666649 middle_crc = 0 data_crc = 0 sig = 8049785851673397625
 -2248> 2017-11-05 21:59:12.841807 7f232c2dc700  5 -- 10.10.1.23:6789/0 >> 10.10.1.21:6789/0 conn(0x7f2342dd4800 :-1 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=1124301 cs=1 l=0). rx mon.0 seq 248515226 0x7f2342beb080 mon_probe(reply 726a42b3-ae4c-4d0b-82c5-0a9065711b5f name ceph01mon quorum 0,1 paxos( fc 112907080 lc 112907738 )) v6
 -2247> 2017-11-05 21:59:12.841878 7f232cadd700 10 _calc_signature seq 123007631 front_crc_ = 2407865230 middle_crc = 0 data_crc = 0 sig = 7765281711407537163
 -2246> 2017-11-05 21:59:12.841926 7f232cadd700  5 -- 10.10.1.23:6789/0 >> 10.10.1.22:6789/0 conn(0x7f2342dd3000 :-1 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=973717 cs=1 l=0). rx mon.1 seq 123007631 0x7f2342bebb80 mon_probe(reply 726a42b3-ae4c-4d0b-82c5-0a9065711b5f name ceph02mon quorum 0,1 paxos( fc 112907080 lc 112907738 )) v6
 -2245> 2017-11-05 21:59:12.884793 7f23302e4700 10 mon.ceph03@2(electing) e14 join_election
 -2244> 2017-11-05 21:59:12.884808 7f23302e4700 10 mon.ceph03@2(electing) e14 _reset
 -2243> 2017-11-05 21:59:12.884811 7f23302e4700 10 mon.ceph03@2(electing) e14 cancel_probe_timeout (none scheduled)
 -2242> 2017-11-05 21:59:12.884813 7f23302e4700 10 mon.ceph03@2(electing) e14 timecheck_finish
 -2241> 2017-11-05 21:59:12.884816 7f23302e4700 15 mon.ceph03@2(electing) e14 health_tick_stop
 -2240> 2017-11-05 21:59:12.884817 7f23302e4700 15 mon.ceph03@2(electing) e14 health_interval_stop
 -2239> 2017-11-05 21:59:12.884819 7f23302e4700 10 mon.ceph03@2(electing) e14 scrub_event_cancel
 -2238> 2017-11-05 21:59:12.884821 7f23302e4700 10 mon.ceph03@2(electing) e14 scrub_reset
 -2237> 2017-11-05 21:59:12.884823 7f23302e4700 10 mon.ceph03@2(electing).paxos(paxos recovering c 112907080..112907738) restart -- canceling timeouts
 -2236> 2017-11-05 21:59:12.884828 7f23302e4700 10 mon.ceph03@2(electing).paxosservice(pgmap 56600067..56600783) restart
 -2235> 2017-11-05 21:59:12.884830 7f23302e4700 10 mon.ceph03@2(electing).paxosservice(mdsmap 1..1) restart
 -2234> 2017-11-05 21:59:12.884832 7f23302e4700 10 mon.ceph03@2(electing).paxosservice(osdmap 503..1105) restart
 -2233> 2017-11-05 21:59:12.884834 7f23302e4700 10 mon.ceph03@2(electing).paxosservice(logm 55596915..55597634) restart
 -2232> 2017-11-05 21:59:12.884836 7f23302e4700 10 mon.ceph03@2(electing).paxosservice(monmap 1..14) restart
 -2231> 2017-11-05 21:59:12.884840 7f23302e4700 10 mon.ceph03@2(electing).paxosservice(auth 26252..26331) restart
 -2230> 2017-11-05 21:59:12.884842 7f23302e4700 10 mon.ceph03@2(electing).paxosservice(mgr 1..394) restart
 -2229> 2017-11-05 21:59:12.884844 7f23302e4700 10 mon.ceph03@2(electing).paxosservice(mgrstat 56609513..56609633) restart
 -2228> 2017-11-05 21:59:12.884845 7f23302e4700 10 mon.ceph03@2(electing).paxosservice(health 1..23) restart
 -2227> 2017-11-05 21:59:12.885270 7f23302e4700  1 -- 10.10.1.23:6789/0 _send_message--> mon.0 10.10.1.21:6789/0 -- election(726a42b3-ae4c-4d0b-82c5-0a9065711b5f propose 339) v7 -- ?+0 0x7f2342cd29c0
 -2226> 2017-11-05 21:59:12.885287 7f23302e4700  1 -- 10.10.1.23:6789/0 --> 10.10.1.21:6789/0 -- election(726a42b3-ae4c-4d0b-82c5-0a9065711b5f propose 339) v7 -- 0x7f2342cd29c0 con 0
 -2225> 2017-11-05 21:59:12.885315 7f23302e4700  1 -- 10.10.1.23:6789/0 _send_message--> mon.1 10.10.1.22:6789/0 -- election(726a42b3-ae4c-4d0b-82c5-0a9065711b5f propose 339) v7 -- ?+0 0x7f2342cd3040
 -2224> 2017-11-05 21:59:12.885325 7f23302e4700  1 -- 10.10.1.23:6789/0 --> 10.10.1.22:6789/0 -- election(726a42b3-ae4c-4d0b-82c5-0a9065711b5f propose 339) v7 -- 0x7f2342cd3040 con 0
 -2223> 2017-11-05 21:59:12.885354 7f232c2dc700 10 _calc_signature seq 5 front_crc_ = 2237315324 middle_crc = 0 data_crc = 0 sig = 12691232392794583066
 -2222> 2017-11-05 21:59:12.885380 7f232c2dc700 20 Putting signature in client message(seq # 5): sig = 12691232392794583066
 -2221> 2017-11-05 21:59:12.885354 7f23302e4700  1 -- 10.10.1.23:6789/0 <== mon.2 10.10.1.23:6789/0 0 ==== log(1 entries from seq 1 at 2017-11-05 21:59:12.841468) v1 ==== 0+0+0 (0 0 0) 0x7f234296e300 con 0x7f23431d6000
 -2220> 2017-11-05 21:59:12.885401 7f23302e4700 10 mon.ceph03@2(electing) e14 _ms_dispatch new session 0x7f2342930a80 MonSession(mon.2 10.10.1.23:6789/0 is open ) features 0x1ffddff8eea4fffb
 -2219> 2017-11-05 21:59:12.885412 7f23302e4700  5 mon.ceph03@2(electing) e14 _ms_dispatch setting monitor caps on this connection
 -2218> 2017-11-05 21:59:12.885407 7f232cadd700 10 _calc_signature seq 3 front_crc_ = 2237315324 middle_crc = 0 data_crc = 0 sig = 53150067798815490
 -2217> 2017-11-05 21:59:12.885415 7f23302e4700 20 mon.ceph03@2(electing) e14  caps allow *
 -2216> 2017-11-05 21:59:12.885416 7f232cadd700 20 Putting signature in client message(seq # 3): sig = 53150067798815490
 -2215> 2017-11-05 21:59:12.885418 7f23302e4700 10 mon.ceph03@2(electing).paxosservice(logm 55596915..55597634) dispatch 0x7f234296e300 log(1 entries from seq 1 at 2017-11-05 21:59:12.841468) v1 from mon.2 10.10.1.23:6789/0 con 0x7f23431d6000
 -2214> 2017-11-05 21:59:12.885427 7f23302e4700  5 mon.ceph03@2(electing).paxos(paxos recovering c 112907080..112907738) is_readable = 0 - now=2017-11-05 21:59:12.885427 lease_expire=0.000000 has v0 lc 112907738
 -2213> 2017-11-05 21:59:12.885433 7f23302e4700 10 mon.ceph03@2(electing).paxosservice(logm 55596915..55597634)  waiting for paxos -> readable (v0)
 -2212> 2017-11-05 21:59:12.885438 7f23302e4700  5 mon.ceph03@2(electing).paxos(paxos recovering c 112907080..112907738) is_readable = 0 - now=2017-11-05 21:59:12.885438 lease_expire=0.000000 has v0 lc 112907738
 -2211> 2017-11-05 21:59:12.885443 7f23302e4700  1 -- 10.10.1.23:6789/0 <== mon.0 10.10.1.21:6789/0 248515226 ==== mon_probe(reply 726a42b3-ae4c-4d0b-82c5-0a9065711b5f name ceph01mon quorum 0,1 paxos( fc 112907080 lc 112907738 )) v6 ==== 494+0+0 (750666649 0 0) 0x7f2342beb080 con 0x7f2342dd4800
 -2210> 2017-11-05 21:59:12.885455 7f23302e4700 20 mon.ceph03@2(electing) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
 -2209> 2017-11-05 21:59:12.885458 7f23302e4700 20 mon.ceph03@2(electing) e14  caps allow *
 -2208> 2017-11-05 21:59:12.885461 7f23302e4700 20 is_capable service=mon command= read on cap allow *
 -2207> 2017-11-05 21:59:12.885463 7f23302e4700 20  allow so far , doing grant allow *
 -2206> 2017-11-05 21:59:12.885464 7f23302e4700 20  allow all
 -2205> 2017-11-05 21:59:12.885465 7f23302e4700 10 mon.ceph03@2(electing) e14 handle_probe mon_probe(reply 726a42b3-ae4c-4d0b-82c5-0a9065711b5f name ceph01mon quorum 0,1 paxos( fc 112907080 lc 112907738 )) v6
 -2204> 2017-11-05 21:59:12.885469 7f23302e4700 10 mon.ceph03@2(electing) e14 handle_probe_reply mon.0 10.10.1.21:6789/0mon_probe(reply 726a42b3-ae4c-4d0b-82c5-0a9065711b5f name ceph01mon quorum 0,1 paxos( fc 112907080 lc 112907738 )) v6
 -2203> 2017-11-05 21:59:12.885473 7f23302e4700 10 mon.ceph03@2(electing) e14  monmap is e14: 3 mons at {ceph01mon=10.10.1.21:6789/0,ceph02mon=10.10.1.22:6789/0,ceph03=10.10.1.23:6789/0}
 -2202> 2017-11-05 21:59:12.885488 7f23302e4700 10 mon.ceph03@2(electing) e14  peer name is ceph01mon
 -2201> 2017-11-05 21:59:12.885498 7f23302e4700  1 -- 10.10.1.23:6789/0 <== mon.1 10.10.1.22:6789/0 123007631 ==== mon_probe(reply 726a42b3-ae4c-4d0b-82c5-0a9065711b5f name ceph02mon quorum 0,1 paxos( fc 112907080 lc 112907738 )) v6 ==== 494+0+0 (2407865230 0 0) 0x7f2342bebb80 con 0x7f2342dd3000
 -2200> 2017-11-05 21:59:12.885506 7f23302e4700 20 mon.ceph03@2(electing) e14 _ms_dispatch existing session 0x7f2342930c00 for mon.1 10.10.1.22:6789/0
 -2199> 2017-11-05 21:59:12.885509 7f23302e4700 20 mon.ceph03@2(electing) e14  caps allow *
 -2198> 2017-11-05 21:59:12.885511 7f23302e4700 20 is_capable service=mon command= read on cap allow *
 -2197> 2017-11-05 21:59:12.885513 7f23302e4700 20  allow so far , doing grant allow *
 -2196> 2017-11-05 21:59:12.885513 7f23302e4700 20  allow all
 -2195> 2017-11-05 21:59:12.885514 7f23302e4700 10 mon.ceph03@2(electing) e14 handle_probe mon_probe(reply 726a42b3-ae4c-4d0b-82c5-0a9065711b5f name ceph02mon quorum 0,1 paxos( fc 112907080 lc 112907738 )) v6
 -2194> 2017-11-05 21:59:12.885518 7f23302e4700 10 mon.ceph03@2(electing) e14 handle_probe_reply mon.1 10.10.1.22:6789/0mon_probe(reply 726a42b3-ae4c-4d0b-82c5-0a9065711b5f name ceph02mon quorum 0,1 paxos( fc 112907080 lc 112907738 )) v6
 -2193> 2017-11-05 21:59:12.885522 7f23302e4700 10 mon.ceph03@2(electing) e14  monmap is e14: 3 mons at {ceph01mon=10.10.1.21:6789/0,ceph02mon=10.10.1.22:6789/0,ceph03=10.10.1.23:6789/0}
 -2192> 2017-11-05 21:59:12.885532 7f23302e4700 10 mon.ceph03@2(electing) e14  peer name is ceph02mon
 -2191> 2017-11-05 21:59:12.896611 7f232d2de700 10 mon.ceph03@2(electing) e14 ms_get_authorizer for mgr
 -2190> 2017-11-05 21:59:12.896688 7f232d2de700 10 cephx keyserverdata: get_caps: name=mon.
 -2189> 2017-11-05 21:59:12.896698 7f232d2de700 10 cephx: build_service_ticket service mgr secret_id 7 ticket_info.ticket.name=mon.
 -2188> 2017-11-05 21:59:12.896957 7f232d2de700  2 -- 10.10.1.23:0/15195 >> 10.10.1.21:6806/22556 conn(0x7f2342dab800 :-1 s=STATE_CONNECTING_WAIT_ACK_SEQ pgs=0 cs=0 l=0)._process_connection got newly_acked_seq 0 vs out_seq 0
 -2187> 2017-11-05 21:59:12.896989 7f232d2de700 10 In get_auth_session_handler for protocol 2
 -2186> 2017-11-05 21:59:12.897028 7f232d2de700 10 _calc_signature seq 1 front_crc_ = 539573588 middle_crc = 0 data_crc = 0 sig = 2604218379037085131
 -2185> 2017-11-05 21:59:12.897039 7f232d2de700 20 Putting signature in client message(seq # 1): sig = 2604218379037085131
 -2184> 2017-11-05 21:59:12.897285 7f232d2de700 10 _calc_signature seq 1 front_crc_ = 2791807819 middle_crc = 0 data_crc = 0 sig = 898952956200682019
 -2183> 2017-11-05 21:59:12.897294 7f232d2de700  5 -- 10.10.1.23:0/15195 >> 10.10.1.21:6806/22556 conn(0x7f2342dab800 :-1 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=9034 cs=1 l=1). rx mgr.54805773 seq 1 0x7f2342cd81c0 mgrconfigure() v1
 -2182> 2017-11-05 21:59:12.897331 7f232eae1700  1 -- 10.10.1.23:0/15195 <== mgr.54805773 10.10.1.21:6806/22556 1 ==== mgrconfigure() v1 ==== 4+0+0 (2791807819 0 0) 0x7f2342cd81c0 con 0x7f2342dab800
 -2181> 2017-11-05 21:59:12.897366 7f232eae1700  4 mgrc handle_mgr_configure stats_period=5
 -2180> 2017-11-05 21:59:12.897614 7f232eae1700  1 -- 10.10.1.23:0/15195 --> 10.10.1.21:6806/22556 -- mgrreport(unknown.ceph03 +135-0 packed 1550) v4 -- 0x7f2342beb600 con 0
 -2179> 2017-11-05 21:59:12.897766 7f232d2de700 10 _calc_signature seq 2 front_crc_ = 2639686472 middle_crc = 0 data_crc = 0 sig = 10048159117889152159
 -2178> 2017-11-05 21:59:12.897783 7f232d2de700 20 Putting signature in client message(seq # 2): sig = 10048159117889152159
 -2177> 2017-11-05 21:59:12.911682 7f232cadd700 10 _calc_signature seq 123007632 front_crc_ = 518921869 middle_crc = 0 data_crc = 0 sig = 16547273314806196539
 -2176> 2017-11-05 21:59:12.911698 7f232cadd700  5 -- 10.10.1.23:6789/0 >> 10.10.1.22:6789/0 conn(0x7f2342dd3000 :-1 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=973717 cs=1 l=0). rx mon.1 seq 123007632 0x7f2342cd3040 election(726a42b3-ae4c-4d0b-82c5-0a9065711b5f propose 343) v7
 -2175> 2017-11-05 21:59:12.911762 7f23302e4700  1 -- 10.10.1.23:6789/0 <== mon.1 10.10.1.22:6789/0 123007632 ==== election(726a42b3-ae4c-4d0b-82c5-0a9065711b5f propose 343) v7 ==== 498+0+0 (518921869 0 0) 0x7f2342cd3040 con 0x7f2342dd3000
 -2174> 2017-11-05 21:59:12.911796 7f23302e4700 20 mon.ceph03@2(electing) e14 _ms_dispatch existing session 0x7f2342930c00 for mon.1 10.10.1.22:6789/0
 -2173> 2017-11-05 21:59:12.911805 7f23302e4700 20 mon.ceph03@2(electing) e14  caps allow *
 -2172> 2017-11-05 21:59:12.911808 7f23302e4700 20 is_capable service=mon command= read on cap allow *
 -2171> 2017-11-05 21:59:12.911811 7f23302e4700 20  allow so far , doing grant allow *
 -2170> 2017-11-05 21:59:12.911812 7f23302e4700 20  allow all
 -2169> 2017-11-05 21:59:12.911814 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
 -2168> 2017-11-05 21:59:12.911815 7f23302e4700 20  allow so far , doing grant allow *
 -2167> 2017-11-05 21:59:12.911817 7f23302e4700 20  allow all
 -2166> 2017-11-05 21:59:12.911839 7f23302e4700  5 mon.ceph03@2(electing).elector(339) handle_propose from mon.1
 -2165> 2017-11-05 21:59:12.911842 7f23302e4700 10 mon.ceph03@2(electing).elector(339) handle_propose required features 153140804152475648 mon_feature_t([kraken,luminous]), peer features 2305244844532236283 mon_feature_t([kraken,luminous])
 -2164> 2017-11-05 21:59:12.911848 7f23302e4700 10 mon.ceph03@2(electing).elector(339) bump_epoch 339 to 343
 -2163> 2017-11-05 21:59:12.920536 7f232c2dc700 10 _calc_signature seq 248515227 front_crc_ = 518921869 middle_crc = 0 data_crc = 0 sig = 326737325304326689
 -2162> 2017-11-05 21:59:12.920553 7f232c2dc700  5 -- 10.10.1.23:6789/0 >> 10.10.1.21:6789/0 conn(0x7f2342dd4800 :-1 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=1124301 cs=1 l=0). rx mon.0 seq 248515227 0x7f2342cd29c0 election(726a42b3-ae4c-4d0b-82c5-0a9065711b5f propose 343) v7
 -2161> 2017-11-05 21:59:12.951319 7f23302e4700 10 mon.ceph03@2(electing) e14 join_election
 -2160> 2017-11-05 21:59:12.951334 7f23302e4700 10 mon.ceph03@2(electing) e14 _reset
 -2159> 2017-11-05 21:59:12.951336 7f23302e4700 10 mon.ceph03@2(electing) e14 cancel_probe_timeout (none scheduled)
 -2158> 2017-11-05 21:59:12.951337 7f23302e4700 10 mon.ceph03@2(electing) e14 timecheck_finish
 -2157> 2017-11-05 21:59:12.951339 7f23302e4700 15 mon.ceph03@2(electing) e14 health_tick_stop
 -2156> 2017-11-05 21:59:12.951340 7f23302e4700 15 mon.ceph03@2(electing) e14 health_interval_stop
 -2155> 2017-11-05 21:59:12.951342 7f23302e4700 10 mon.ceph03@2(electing) e14 scrub_event_cancel
 -2154> 2017-11-05 21:59:12.951343 7f23302e4700 10 mon.ceph03@2(electing) e14 scrub_reset
 -2153> 2017-11-05 21:59:12.951346 7f23302e4700 10 mon.ceph03@2(electing).paxos(paxos recovering c 112907080..112907738) restart -- canceling timeouts
 -2152> 2017-11-05 21:59:12.951350 7f23302e4700 10 mon.ceph03@2(electing).paxosservice(pgmap 56600067..56600783) restart
 -2151> 2017-11-05 21:59:12.951353 7f23302e4700 10 mon.ceph03@2(electing).paxosservice(mdsmap 1..1) restart
 -2150> 2017-11-05 21:59:12.951355 7f23302e4700 10 mon.ceph03@2(electing).paxosservice(osdmap 503..1105) restart
 -2149> 2017-11-05 21:59:12.951356 7f23302e4700 10 mon.ceph03@2(electing).paxosservice(logm 55596915..55597634) restart
 -2148> 2017-11-05 21:59:12.951358 7f23302e4700 10 mon.ceph03@2(electing).paxosservice(monmap 1..14) restart
 -2147> 2017-11-05 21:59:12.951359 7f23302e4700 10 mon.ceph03@2(electing).paxosservice(auth 26252..26331) restart
 -2146> 2017-11-05 21:59:12.951361 7f23302e4700 10 mon.ceph03@2(electing).paxosservice(mgr 1..394) restart
 -2145> 2017-11-05 21:59:12.951363 7f23302e4700 10 mon.ceph03@2(electing).paxosservice(mgrstat 56609513..56609633) restart
 -2144> 2017-11-05 21:59:12.951365 7f23302e4700 10 mon.ceph03@2(electing).paxosservice(health 1..23) restart
 -2143> 2017-11-05 21:59:12.951375 7f23302e4700  5 mon.ceph03@2(electing).elector(343) defer to 1
 -2142> 2017-11-05 21:59:12.951660 7f23302e4700  1 -- 10.10.1.23:6789/0 _send_message--> mon.1 10.10.1.22:6789/0 -- election(726a42b3-ae4c-4d0b-82c5-0a9065711b5f ack 343) v7 -- ?+0 0x7f2342cd2d00
 -2141> 2017-11-05 21:59:12.951676 7f23302e4700  1 -- 10.10.1.23:6789/0 --> 10.10.1.22:6789/0 -- election(726a42b3-ae4c-4d0b-82c5-0a9065711b5f ack 343) v7 -- 0x7f2342cd2d00 con 0
 -2140> 2017-11-05 21:59:12.951711 7f23302e4700  1 -- 10.10.1.23:6789/0 <== mon.0 10.10.1.21:6789/0 248515227 ==== election(726a42b3-ae4c-4d0b-82c5-0a9065711b5f propose 343) v7 ==== 498+0+0 (518921869 0 0) 0x7f2342cd29c0 con 0x7f2342dd4800
 -2139> 2017-11-05 21:59:12.951725 7f23302e4700 20 mon.ceph03@2(electing) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
 -2138> 2017-11-05 21:59:12.951734 7f23302e4700 20 mon.ceph03@2(electing) e14  caps allow *
 -2137> 2017-11-05 21:59:12.951737 7f23302e4700 20 is_capable service=mon command= read on cap allow *
 -2136> 2017-11-05 21:59:12.951739 7f23302e4700 20  allow so far , doing grant allow *
 -2135> 2017-11-05 21:59:12.951740 7f23302e4700 20  allow all
 -2134> 2017-11-05 21:59:12.951741 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
 -2133> 2017-11-05 21:59:12.951743 7f23302e4700 20  allow so far , doing grant allow *
 -2132> 2017-11-05 21:59:12.951743 7f23302e4700 20  allow all
 -2131> 2017-11-05 21:59:12.951760 7f23302e4700  5 mon.ceph03@2(electing).elector(343) handle_propose from mon.0
 -2130> 2017-11-05 21:59:12.951762 7f23302e4700 10 mon.ceph03@2(electing).elector(343) handle_propose required features 153140804152475648 mon_feature_t([kraken,luminous]), peer features 2305244844532236283 mon_feature_t([kraken,luminous])
 -2129> 2017-11-05 21:59:12.951766 7f23302e4700  5 mon.ceph03@2(electing).elector(343) defer to 0
 -2128> 2017-11-05 21:59:12.951772 7f232cadd700 10 _calc_signature seq 4 front_crc_ = 2543563982 middle_crc = 0 data_crc = 0 sig = 12003399029982092639
 -2127> 2017-11-05 21:59:12.951781 7f232cadd700 20 Putting signature in client message(seq # 4): sig = 12003399029982092639
 -2126> 2017-11-05 21:59:12.951992 7f23302e4700  1 -- 10.10.1.23:6789/0 _send_message--> mon.0 10.10.1.21:6789/0 -- election(726a42b3-ae4c-4d0b-82c5-0a9065711b5f ack 343) v7 -- ?+0 0x7f2342cd3040
 -2125> 2017-11-05 21:59:12.952006 7f23302e4700  1 -- 10.10.1.23:6789/0 --> 10.10.1.21:6789/0 -- election(726a42b3-ae4c-4d0b-82c5-0a9065711b5f ack 343) v7 -- 0x7f2342cd3040 con 0
 -2124> 2017-11-05 21:59:12.952081 7f232c2dc700 10 _calc_signature seq 6 front_crc_ = 2543563982 middle_crc = 0 data_crc = 0 sig = 9154992000545278758
 -2123> 2017-11-05 21:59:12.952098 7f232c2dc700 20 Putting signature in client message(seq # 6): sig = 9154992000545278758
 -2122> 2017-11-05 21:59:12.987082 7f232c2dc700 10 _calc_signature seq 248515228 front_crc_ = 2232557517 middle_crc = 0 data_crc = 0 sig = 9253898422206336076
 -2121> 2017-11-05 21:59:12.987107 7f232c2dc700  5 -- 10.10.1.23:6789/0 >> 10.10.1.21:6789/0 conn(0x7f2342dd4800 :-1 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=1124301 cs=1 l=0). rx mon.0 seq 248515228 0x7f2342cd3040 election(726a42b3-ae4c-4d0b-82c5-0a9065711b5f victory 344) v7
 -2120> 2017-11-05 21:59:12.987173 7f23302e4700  1 -- 10.10.1.23:6789/0 <== mon.0 10.10.1.21:6789/0 248515228 ==== election(726a42b3-ae4c-4d0b-82c5-0a9065711b5f victory 344) v7 ==== 39085+0+0 (2232557517 0 0) 0x7f2342cd3040 con 0x7f2342dd4800
 -2119> 2017-11-05 21:59:12.987206 7f23302e4700 20 mon.ceph03@2(electing) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
 -2118> 2017-11-05 21:59:12.987215 7f23302e4700 20 mon.ceph03@2(electing) e14  caps allow *
 -2117> 2017-11-05 21:59:12.987218 7f23302e4700 20 is_capable service=mon command= read on cap allow *
 -2116> 2017-11-05 21:59:12.987220 7f23302e4700 20  allow so far , doing grant allow *
 -2115> 2017-11-05 21:59:12.987221 7f23302e4700 20  allow all
 -2114> 2017-11-05 21:59:12.987223 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
 -2113> 2017-11-05 21:59:12.987224 7f23302e4700 20  allow so far , doing grant allow *
 -2112> 2017-11-05 21:59:12.987225 7f23302e4700 20  allow all
 -2111> 2017-11-05 21:59:12.987243 7f23302e4700  5 mon.ceph03@2(electing).elector(343) handle_victory from mon.0 quorum_features 2305244844532236283 mon_feature_t([kraken,luminous])
 -2110> 2017-11-05 21:59:12.987248 7f23302e4700 10 mon.ceph03@2(electing).elector(343) bump_epoch 343 to 344
 -2109> 2017-11-05 21:59:13.029008 7f23302e4700 10 mon.ceph03@2(electing) e14 join_election
 -2108> 2017-11-05 21:59:13.029034 7f23302e4700 10 mon.ceph03@2(electing) e14 _reset
 -2107> 2017-11-05 21:59:13.029036 7f23302e4700 10 mon.ceph03@2(electing) e14 cancel_probe_timeout (none scheduled)
 -2106> 2017-11-05 21:59:13.029038 7f23302e4700 10 mon.ceph03@2(electing) e14 timecheck_finish
 -2105> 2017-11-05 21:59:13.029040 7f23302e4700 15 mon.ceph03@2(electing) e14 health_tick_stop
 -2104> 2017-11-05 21:59:13.029042 7f23302e4700 15 mon.ceph03@2(electing) e14 health_interval_stop
 -2103> 2017-11-05 21:59:13.029043 7f23302e4700 10 mon.ceph03@2(electing) e14 scrub_event_cancel
 -2102> 2017-11-05 21:59:13.029045 7f23302e4700 10 mon.ceph03@2(electing) e14 scrub_reset
 -2101> 2017-11-05 21:59:13.029047 7f23302e4700 10 mon.ceph03@2(electing).paxos(paxos recovering c 112907080..112907738) restart -- canceling timeouts
 -2100> 2017-11-05 21:59:13.029052 7f23302e4700 10 mon.ceph03@2(electing).paxosservice(pgmap 56600067..56600783) restart
 -2099> 2017-11-05 21:59:13.029056 7f23302e4700 10 mon.ceph03@2(electing).paxosservice(mdsmap 1..1) restart
 -2098> 2017-11-05 21:59:13.029058 7f23302e4700 10 mon.ceph03@2(electing).paxosservice(osdmap 503..1105) restart
 -2097> 2017-11-05 21:59:13.029060 7f23302e4700 10 mon.ceph03@2(electing).paxosservice(logm 55596915..55597634) restart
 -2096> 2017-11-05 21:59:13.029062 7f23302e4700 10 mon.ceph03@2(electing).paxosservice(monmap 1..14) restart
 -2095> 2017-11-05 21:59:13.029064 7f23302e4700 10 mon.ceph03@2(electing).paxosservice(auth 26252..26331) restart
 -2094> 2017-11-05 21:59:13.029065 7f23302e4700 10 mon.ceph03@2(electing).paxosservice(mgr 1..394) restart
 -2093> 2017-11-05 21:59:13.029067 7f23302e4700 10 mon.ceph03@2(electing).paxosservice(mgrstat 56609513..56609633) restart
 -2092> 2017-11-05 21:59:13.029069 7f23302e4700 10 mon.ceph03@2(electing).paxosservice(health 1..23) restart
 -2091> 2017-11-05 21:59:13.029075 7f23302e4700 10 mon.ceph03@2(peon) e14 lose_election, epoch 344 leader is mon0 quorum is 0,1,2 features are 2305244844532236283 mon_features are mon_feature_t([kraken,luminous])
 -2090> 2017-11-05 21:59:13.029083 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos recovering c 112907080..112907738) peon_init -- i am a peon
 -2089> 2017-11-05 21:59:13.029086 7f23302e4700 20 mon.ceph03@2(peon).paxos(paxos recovering c 112907080..112907738) reset_lease_timeout - setting timeout event
 -2088> 2017-11-05 21:59:13.029092 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(pgmap 56600067..56600783) election_finished
 -2087> 2017-11-05 21:59:13.029094 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(pgmap 56600067..56600783) _active - not active
 -2086> 2017-11-05 21:59:13.029097 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mdsmap 1..1) election_finished
 -2085> 2017-11-05 21:59:13.029099 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mdsmap 1..1) _active - not active
 -2084> 2017-11-05 21:59:13.029101 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(osdmap 503..1105) election_finished
 -2083> 2017-11-05 21:59:13.029102 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(osdmap 503..1105) _active - not active
 -2082> 2017-11-05 21:59:13.029104 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(logm 55596915..55597634) election_finished
 -2081> 2017-11-05 21:59:13.029106 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(logm 55596915..55597634) _active - not active
 -2080> 2017-11-05 21:59:13.029109 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(monmap 1..14) election_finished
 -2079> 2017-11-05 21:59:13.029111 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(monmap 1..14) _active - not active
 -2078> 2017-11-05 21:59:13.029113 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(auth 26252..26331) election_finished
 -2077> 2017-11-05 21:59:13.029114 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(auth 26252..26331) _active - not active
 -2076> 2017-11-05 21:59:13.029116 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgr 1..394) election_finished
 -2075> 2017-11-05 21:59:13.029118 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgr 1..394) _active - not active
 -2074> 2017-11-05 21:59:13.029119 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgrstat 56609513..56609633) election_finished
 -2073> 2017-11-05 21:59:13.029121 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgrstat 56609513..56609633) _active - not active
 -2072> 2017-11-05 21:59:13.029123 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(health 1..23) election_finished
 -2071> 2017-11-05 21:59:13.029124 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(health 1..23) _active - not active
 -2070> 2017-11-05 21:59:13.029126 7f23302e4700 10 mon.ceph03@2(peon).data_health(344) start_epoch epoch 344
 -2069> 2017-11-05 21:59:13.029133 7f23302e4700  5 mon.ceph03@2(peon) e14 apply_quorum_to_compatset_features
 -2068> 2017-11-05 21:59:13.029137 7f23302e4700  5 mon.ceph03@2(peon) e14 apply_monmap_to_compatset_features
 -2067> 2017-11-05 21:59:13.029140 7f23302e4700 10 mon.ceph03@2(peon) e14 timecheck_finish
 -2066> 2017-11-05 21:59:13.029141 7f23302e4700 10 mon.ceph03@2(peon) e14 resend_routed_requests
 -2065> 2017-11-05 21:59:13.029144 7f23302e4700 10 mon.ceph03@2(peon) e14 register_cluster_logger
 -2064> 2017-11-05 21:59:13.256981 7f232c2dc700 10 _calc_signature seq 248515229 front_crc_ = 3785316099 middle_crc = 0 data_crc = 0 sig = 15208736839717074848
 -2063> 2017-11-05 21:59:13.256998 7f232c2dc700  5 -- 10.10.1.23:6789/0 >> 10.10.1.21:6789/0 conn(0x7f2342dd4800 :-1 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=1124301 cs=1 l=0). rx mon.0 seq 248515229 0x7f2342969200 paxos(collect lc 112907738 fc 112907080 pn 13700 opn 0) v4
 -2062> 2017-11-05 21:59:13.257038 7f232c2dc700 10 _calc_signature seq 248515230 front_crc_ = 2465577770 middle_crc = 0 data_crc = 0 sig = 4557865954043635196
 -2061> 2017-11-05 21:59:13.257046 7f232c2dc700  5 -- 10.10.1.23:6789/0 >> 10.10.1.21:6789/0 conn(0x7f2342dd4800 :-1 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=1124301 cs=1 l=0). rx mon.0 seq 248515230 0x7f234296ed00 time_check( ping e 344 r 1 ) v1
 -2060> 2017-11-05 21:59:13.257036 7f23302e4700  1 -- 10.10.1.23:6789/0 <== mon.0 10.10.1.21:6789/0 248515229 ==== paxos(collect lc 112907738 fc 112907080 pn 13700 opn 0) v4 ==== 84+0+0 (3785316099 0 0) 0x7f2342969200 con 0x7f2342dd4800
 -2059> 2017-11-05 21:59:13.257066 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
 -2058> 2017-11-05 21:59:13.257090 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
 -2057> 2017-11-05 21:59:13.257098 7f23302e4700 20 is_capable service=mon command= read on cap allow *
 -2056> 2017-11-05 21:59:13.257101 7f23302e4700 20  allow so far , doing grant allow *
 -2055> 2017-11-05 21:59:13.257103 7f23302e4700 20  allow all
 -2054> 2017-11-05 21:59:13.257106 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
 -2053> 2017-11-05 21:59:13.257108 7f23302e4700 20  allow so far , doing grant allow *
 -2052> 2017-11-05 21:59:13.257110 7f23302e4700 20  allow all
 -2051> 2017-11-05 21:59:13.257115 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos recovering c 112907080..112907738) handle_collect paxos(collect lc 112907738 fc 112907080 pn 13700 opn 0) v4
 -2050> 2017-11-05 21:59:13.257126 7f23302e4700 20 mon.ceph03@2(peon).paxos(paxos recovering c 112907080..112907738) reset_lease_timeout - setting timeout event
 -2049> 2017-11-05 21:59:13.257133 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos recovering c 112907080..112907738) accepting pn 13700 from 0
 -2048> 2017-11-05 21:59:13.294990 7f23302e4700  1 -- 10.10.1.23:6789/0 --> 10.10.1.21:6789/0 -- paxos(last lc 112907738 fc 112907080 pn 13700 opn 0) v4 -- 0x7f2342969500 con 0
 -2047> 2017-11-05 21:59:13.295034 7f23302e4700  1 -- 10.10.1.23:6789/0 <== mon.0 10.10.1.21:6789/0 248515230 ==== time_check( ping e 344 r 1 ) v1 ==== 36+0+0 (2465577770 0 0) 0x7f234296ed00 con 0x7f2342dd4800
 -2046> 2017-11-05 21:59:13.295049 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
 -2045> 2017-11-05 21:59:13.295058 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
 -2044> 2017-11-05 21:59:13.295061 7f23302e4700 20 is_capable service=mon command= read on cap allow *
 -2043> 2017-11-05 21:59:13.295064 7f23302e4700 20  allow so far , doing grant allow *
 -2042> 2017-11-05 21:59:13.295065 7f23302e4700 20  allow all
 -2041> 2017-11-05 21:59:13.295067 7f23302e4700 10 mon.ceph03@2(peon) e14 handle_timecheck time_check( ping e 344 r 1 ) v1
 -2040> 2017-11-05 21:59:13.295070 7f23302e4700 10 mon.ceph03@2(peon) e14 handle_timecheck_peon time_check( ping e 344 r 1 ) v1
 -2039> 2017-11-05 21:59:13.295065 7f232c2dc700 10 _calc_signature seq 7 front_crc_ = 769890768 middle_crc = 0 data_crc = 0 sig = 16174551769322836280
 -2038> 2017-11-05 21:59:13.295076 7f232c2dc700 20 Putting signature in client message(seq # 7): sig = 16174551769322836280
 -2037> 2017-11-05 21:59:13.295074 7f23302e4700 10 mon.ceph03@2(peon) e14 handle_timecheck_peon send time_check( ping e 344 r 1 ) v1 to mon.0 10.10.1.21:6789/0
 -2036> 2017-11-05 21:59:13.295079 7f23302e4700  1 -- 10.10.1.23:6789/0 --> 10.10.1.21:6789/0 -- time_check( pong e 344 r 1 ts 2017-11-05 21:59:13.295073 ) v1 -- 0x7f234296e580 con 0
 -2035> 2017-11-05 21:59:13.295116 7f232c2dc700 10 _calc_signature seq 8 front_crc_ = 2310135063 middle_crc = 0 data_crc = 0 sig = 16753175160622485731
 -2034> 2017-11-05 21:59:13.295121 7f232c2dc700 20 Putting signature in client message(seq # 8): sig = 16753175160622485731
 -2033> 2017-11-05 21:59:13.295420 7f232c2dc700 10 _calc_signature seq 248515231 front_crc_ = 2060387759 middle_crc = 0 data_crc = 0 sig = 15970751659405569903
 -2032> 2017-11-05 21:59:13.295433 7f232c2dc700  5 -- 10.10.1.23:6789/0 >> 10.10.1.21:6789/0 conn(0x7f2342dd4800 :-1 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=1124301 cs=1 l=0). rx mon.0 seq 248515231 0x7f2342969500 paxos(lease lc 112907738 fc 112907080 pn 0 opn 0) v4
 -2031> 2017-11-05 21:59:13.295494 7f23302e4700  1 -- 10.10.1.23:6789/0 <== mon.0 10.10.1.21:6789/0 248515231 ==== paxos(lease lc 112907738 fc 112907080 pn 0 opn 0) v4 ==== 84+0+0 (2060387759 0 0) 0x7f2342969500 con 0x7f2342dd4800
 -2030> 2017-11-05 21:59:13.295525 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
 -2029> 2017-11-05 21:59:13.295534 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
 -2028> 2017-11-05 21:59:13.295537 7f23302e4700 20 is_capable service=mon command= read on cap allow *
 -2027> 2017-11-05 21:59:13.295540 7f23302e4700 20  allow so far , doing grant allow *
 -2026> 2017-11-05 21:59:13.295541 7f23302e4700 20  allow all
 -2025> 2017-11-05 21:59:13.295543 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
 -2024> 2017-11-05 21:59:13.295545 7f23302e4700 20  allow so far , doing grant allow *
 -2023> 2017-11-05 21:59:13.295546 7f23302e4700 20  allow all
 -2022> 2017-11-05 21:59:13.295551 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907738) handle_lease on 112907738 now 2017-11-05 21:59:18.293587
 -2021> 2017-11-05 21:59:13.295564 7f23302e4700  1 -- 10.10.1.23:6789/0 --> 10.10.1.21:6789/0 -- paxos(lease_ack lc 112907738 fc 112907080 pn 0 opn 0) v4 -- 0x7f2342969200 con 0
 -2020> 2017-11-05 21:59:13.295580 7f23302e4700 20 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907738) reset_lease_timeout - setting timeout event
 -2019> 2017-11-05 21:59:13.295589 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(pgmap 56600067..56600783) _active
 -2018> 2017-11-05 21:59:13.295591 7f23302e4700  7 mon.ceph03@2(peon).paxosservice(pgmap 56600067..56600783) _active we are not the leader, hence we propose nothing!
 -2017> 2017-11-05 21:59:13.295595 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mdsmap 1..1) _active
 -2016> 2017-11-05 21:59:13.295597 7f23302e4700  7 mon.ceph03@2(peon).paxosservice(mdsmap 1..1) _active we are not the leader, hence we propose nothing!
 -2015> 2017-11-05 21:59:13.295600 7f23302e4700 10 mon.ceph03@2(peon).mds e1
 -2014> 2017-11-05 21:59:13.295604 7f23302e4700 10 mon.ceph03@2(peon).mds e1 update_logger
 -2013> 2017-11-05 21:59:13.295607 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(osdmap 503..1105) _active
 -2012> 2017-11-05 21:59:13.295609 7f23302e4700  7 mon.ceph03@2(peon).paxosservice(osdmap 503..1105) _active we are not the leader, hence we propose nothing!
 -2011> 2017-11-05 21:59:13.295611 7f23302e4700 10 mon.ceph03@2(peon).osd e1105 update_logger
 -2010> 2017-11-05 21:59:13.295616 7f23302e4700 10 mon.ceph03@2(peon).osd e1105 take_all_failures on 0 osds
 -2009> 2017-11-05 21:59:13.295614 7f232c2dc700 10 _calc_signature seq 9 front_crc_ = 3465704679 middle_crc = 0 data_crc = 0 sig = 14492575556736995119
 -2008> 2017-11-05 21:59:13.295618 7f23302e4700 10 mon.ceph03@2(peon).osd e1105 start_mapping canceling previous mapping_job 0x7f2342d4ae00
 -2007> 2017-11-05 21:59:13.295620 7f232c2dc700 20 Putting signature in client message(seq # 9): sig = 14492575556736995119
 -2006> 2017-11-05 21:59:13.295628 7f23302e4700 20 queue 0x7f2342dddc60 1 [0,512)
 -2005> 2017-11-05 21:59:13.295633 7f23302e4700 20 queue 0x7f2342dddc60 2 [0,128)
 -2004> 2017-11-05 21:59:13.295639 7f23302e4700 20 queue 0x7f2342dddc60 4 [0,128)
 -2003> 2017-11-05 21:59:13.295641 7f23302e4700 10 mon.ceph03@2(peon).osd e1105 start_mapping started mapping job 0x7f2342dddc60 at 2017-11-05 21:59:13.295621
 -2002> 2017-11-05 21:59:13.295646 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(logm 55596915..55597634) _active
 -2001> 2017-11-05 21:59:13.295648 7f23302e4700  7 mon.ceph03@2(peon).paxosservice(logm 55596915..55597634) _active we are not the leader, hence we propose nothing!
 -2000> 2017-11-05 21:59:13.295650 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(monmap 1..14) _active
 -1999> 2017-11-05 21:59:13.295651 7f23302e4700  7 mon.ceph03@2(peon).paxosservice(monmap 1..14) _active we are not the leader, hence we propose nothing!
 -1998> 2017-11-05 21:59:13.295653 7f23302e4700  5 mon.ceph03@2(peon).monmap v14 apply_mon_features wait for service to be writeable
 -1997> 2017-11-05 21:59:13.295656 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(auth 26252..26331) _active
 -1996> 2017-11-05 21:59:13.295657 7f23302e4700  7 mon.ceph03@2(peon).paxosservice(auth 26252..26331) _active we are not the leader, hence we propose nothing!
 -1995> 2017-11-05 21:59:13.295659 7f23302e4700 10 mon.ceph03@2(peon).auth v26331 AuthMonitor::on_active()
 -1994> 2017-11-05 21:59:13.295654 7f23312e6700 20 _process 0x7f2342dddc60 1 [0,512)
 -1993> 2017-11-05 21:59:13.295663 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgr 1..394) _active
 -1992> 2017-11-05 21:59:13.295664 7f23302e4700  7 mon.ceph03@2(peon).paxosservice(mgr 1..394) _active we are not the leader, hence we propose nothing!
 -1991> 2017-11-05 21:59:13.295666 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgrstat 56609513..56609633) _active
 -1990> 2017-11-05 21:59:13.295668 7f23302e4700  7 mon.ceph03@2(peon).paxosservice(mgrstat 56609513..56609633) _active we are not the leader, hence we propose nothing!
 -1989> 2017-11-05 21:59:13.295670 7f23302e4700 20 mon.ceph03@2(peon).mgrstat update_logger
 -1988> 2017-11-05 21:59:13.295674 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(health 1..23) _active
 -1987> 2017-11-05 21:59:13.295676 7f23302e4700  7 mon.ceph03@2(peon).paxosservice(health 1..23) _active we are not the leader, hence we propose nothing!
 -1986> 2017-11-05 21:59:13.295678 7f23302e4700  5 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907738) is_readable = 1 - now=2017-11-05 21:59:13.295679 lease_expire=2017-11-05 21:59:18.293587 has v0 lc 112907738
 -1985> 2017-11-05 21:59:13.295686 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(logm 55596915..55597634) dispatch 0x7f234296e300 log(1 entries from seq 1 at 2017-11-05 21:59:12.841468) v1 from mon.2 10.10.1.23:6789/0 con 0x7f23431d6000
 -1984> 2017-11-05 21:59:13.295685 7f2331ae7700 20 _process 0x7f2342dddc60 2 [0,128)
 -1983> 2017-11-05 21:59:13.295694 7f23302e4700  5 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907738) is_readable = 1 - now=2017-11-05 21:59:13.295694 lease_expire=2017-11-05 21:59:18.293587 has v0 lc 112907738
 -1982> 2017-11-05 21:59:13.295695 7f2330ae5700 20 _process 0x7f2342dddc60 4 [0,128)
 -1981> 2017-11-05 21:59:13.295700 7f23302e4700 10 mon.ceph03@2(peon).log v55597634 preprocess_query log(1 entries from seq 1 at 2017-11-05 21:59:12.841468) v1 from mon.2 10.10.1.23:6789/0
 -1980> 2017-11-05 21:59:13.295706 7f23302e4700 10 mon.ceph03@2(peon).log v55597634 preprocess_log log(1 entries from seq 1 at 2017-11-05 21:59:12.841468) v1 from mon.2
 -1979> 2017-11-05 21:59:13.295711 7f23302e4700 20 is_capable service=log command= write on cap allow *
 -1978> 2017-11-05 21:59:13.295713 7f23302e4700 20  allow so far , doing grant allow *
 -1977> 2017-11-05 21:59:13.295714 7f23302e4700 20  allow all
 -1976> 2017-11-05 21:59:13.295732 7f23302e4700 10 mon.ceph03@2(peon) e14 forward_request 1 request log(1 entries from seq 1 at 2017-11-05 21:59:12.841468) v1 features 2305244844532236283
 -1975> 2017-11-05 21:59:13.295744 7f23302e4700  1 -- 10.10.1.23:6789/0 _send_message--> mon.0 10.10.1.21:6789/0 -- forward(log(1 entries from seq 1 at 2017-11-05 21:59:12.841468) v1 caps allow * tid 1 con_features 2305244844532236283) v3 -- ?+0 0x7f234296ed00
 -1974> 2017-11-05 21:59:13.295753 7f23302e4700  1 -- 10.10.1.23:6789/0 --> 10.10.1.21:6789/0 -- forward(log(1 entries from seq 1 at 2017-11-05 21:59:12.841468) v1 caps allow * tid 1 con_features 2305244844532236283) v3 -- 0x7f234296ed00 con 0
 -1973> 2017-11-05 21:59:13.295799 7f232c2dc700 10 _calc_signature seq 10 front_crc_ = 2470816670 middle_crc = 0 data_crc = 0 sig = 2328889181983519027
 -1972> 2017-11-05 21:59:13.295804 7f232c2dc700 20 Putting signature in client message(seq # 10): sig = 2328889181983519027
 -1971> 2017-11-05 21:59:13.297490 7f23312e6700 10 osdmap epoch 1105 mapping took 0.001868 seconds
 -1970> 2017-11-05 21:59:13.297500 7f23312e6700 10 mon.ceph03@2(peon).osd e1105 update_creating_pgs 0 pgs creating, 0 pools in queue
 -1969> 2017-11-05 21:59:13.440996 7f232c2dc700 10 _calc_signature seq 248515232 front_crc_ = 3060764253 middle_crc = 0 data_crc = 0 sig = 1336745259145746491
 -1968> 2017-11-05 21:59:13.441013 7f232c2dc700  5 -- 10.10.1.23:6789/0 >> 10.10.1.21:6789/0 conn(0x7f2342dd4800 :-1 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=1124301 cs=1 l=0). rx mon.0 seq 248515232 0x7f2342969200 paxos(begin lc 112907738 fc 0 pn 13700 opn 0) v4
 -1967> 2017-11-05 21:59:13.441058 7f232c2dc700 10 _calc_signature seq 248515233 front_crc_ = 2324920290 middle_crc = 0 data_crc = 0 sig = 17879750372088187599
 -1966> 2017-11-05 21:59:13.441069 7f232c2dc700  5 -- 10.10.1.23:6789/0 >> 10.10.1.21:6789/0 conn(0x7f2342dd4800 :-1 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=1124301 cs=1 l=0). rx mon.0 seq 248515233 0x7f234296ed00 time_check( report e 344 r 2 #skews 3 #latencies 3 ) v1
 -1965> 2017-11-05 21:59:13.441076 7f23302e4700  1 -- 10.10.1.23:6789/0 <== mon.0 10.10.1.21:6789/0 248515232 ==== paxos(begin lc 112907738 fc 0 pn 13700 opn 0) v4 ==== 1693+0+0 (3060764253 0 0) 0x7f2342969200 con 0x7f2342dd4800
 -1964> 2017-11-05 21:59:13.441121 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
 -1963> 2017-11-05 21:59:13.441131 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
 -1962> 2017-11-05 21:59:13.441134 7f23302e4700 20 is_capable service=mon command= read on cap allow *
 -1961> 2017-11-05 21:59:13.441137 7f23302e4700 20  allow so far , doing grant allow *
 -1960> 2017-11-05 21:59:13.441138 7f23302e4700 20  allow all
 -1959> 2017-11-05 21:59:13.441140 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
 -1958> 2017-11-05 21:59:13.441141 7f23302e4700 20  allow so far , doing grant allow *
 -1957> 2017-11-05 21:59:13.441142 7f23302e4700 20  allow all
 -1956> 2017-11-05 21:59:13.441148 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907738) handle_begin paxos(begin lc 112907738 fc 0 pn 13700 opn 0) v4
 -1955> 2017-11-05 21:59:13.441155 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos updating c 112907080..112907738) accepting value for 112907739 pn 13700
 -1954> 2017-11-05 21:59:13.483260 7f23302e4700  1 -- 10.10.1.23:6789/0 --> 10.10.1.21:6789/0 -- paxos(accept lc 112907738 fc 0 pn 13700 opn 0) v4 -- 0x7f2342969500 con 0
 -1953> 2017-11-05 21:59:13.483313 7f23302e4700  1 -- 10.10.1.23:6789/0 <== mon.0 10.10.1.21:6789/0 248515233 ==== time_check( report e 344 r 2 #skews 3 #latencies 3 ) v1 ==== 348+0+0 (2324920290 0 0) 0x7f234296ed00 con 0x7f2342dd4800
 -1952> 2017-11-05 21:59:13.483332 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
 -1951> 2017-11-05 21:59:13.483342 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
 -1950> 2017-11-05 21:59:13.483345 7f23302e4700 20 is_capable service=mon command= read on cap allow *
 -1949> 2017-11-05 21:59:13.483347 7f23302e4700 20  allow so far , doing grant allow *
 -1948> 2017-11-05 21:59:13.483349 7f23302e4700 20  allow all
 -1947> 2017-11-05 21:59:13.483351 7f23302e4700 10 mon.ceph03@2(peon) e14 handle_timecheck time_check( report e 344 r 2 #skews 3 #latencies 3 ) v1
 -1946> 2017-11-05 21:59:13.483355 7f23302e4700 10 mon.ceph03@2(peon) e14 handle_timecheck_peon time_check( report e 344 r 2 #skews 3 #latencies 3 ) v1
 -1945> 2017-11-05 21:59:13.483370 7f232c2dc700 10 _calc_signature seq 11 front_crc_ = 4281357303 middle_crc = 0 data_crc = 0 sig = 5973436363658062779
 -1944> 2017-11-05 21:59:13.483381 7f232c2dc700 20 Putting signature in client message(seq # 11): sig = 5973436363658062779
 -1943> 2017-11-05 21:59:13.518562 7f232c2dc700 10 _calc_signature seq 248515234 front_crc_ = 2766364624 middle_crc = 0 data_crc = 0 sig = 1303804729337060943
 -1942> 2017-11-05 21:59:13.518577 7f232c2dc700  5 -- 10.10.1.23:6789/0 >> 10.10.1.21:6789/0 conn(0x7f2342dd4800 :-1 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=1124301 cs=1 l=0). rx mon.0 seq 248515234 0x7f2342969500 paxos(commit lc 112907739 fc 0 pn 13700 opn 0) v4
 -1941> 2017-11-05 21:59:13.518643 7f23302e4700  1 -- 10.10.1.23:6789/0 <== mon.0 10.10.1.21:6789/0 248515234 ==== paxos(commit lc 112907739 fc 0 pn 13700 opn 0) v4 ==== 1693+0+0 (2766364624 0 0) 0x7f2342969500 con 0x7f2342dd4800
 -1940> 2017-11-05 21:59:13.518675 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
 -1939> 2017-11-05 21:59:13.518684 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
 -1938> 2017-11-05 21:59:13.518687 7f23302e4700 20 is_capable service=mon command= read on cap allow *
 -1937> 2017-11-05 21:59:13.518696 7f23302e4700 20  allow so far , doing grant allow *
 -1936> 2017-11-05 21:59:13.518697 7f23302e4700 20  allow all
 -1935> 2017-11-05 21:59:13.518699 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
 -1934> 2017-11-05 21:59:13.518700 7f23302e4700 20  allow so far , doing grant allow *
 -1933> 2017-11-05 21:59:13.518701 7f23302e4700 20  allow all
 -1932> 2017-11-05 21:59:13.518708 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos updating c 112907080..112907738) handle_commit on 112907739
 -1931> 2017-11-05 21:59:13.518714 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos updating c 112907080..112907739) store_state [112907739..112907739]
 -1930> 2017-11-05 21:59:13.518793 7f232c2dc700 10 _calc_signature seq 248515235 front_crc_ = 2163361469 middle_crc = 0 data_crc = 0 sig = 2435996645971876188
 -1929> 2017-11-05 21:59:13.518804 7f232c2dc700  5 -- 10.10.1.23:6789/0 >> 10.10.1.21:6789/0 conn(0x7f2342dd4800 :-1 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=1124301 cs=1 l=0). rx mon.0 seq 248515235 0x7f2342969b00 paxos(lease lc 112907739 fc 112907080 pn 0 opn 0) v4
 -1928> 2017-11-05 21:59:13.560989 7f23302e4700 10 mon.ceph03@2(peon) e14 refresh_from_paxos
 -1927> 2017-11-05 21:59:13.561022 7f23302e4700 10 mon.ceph03@2(peon) e14 refresh_from_paxos no cluster_fingerprint
 -1926> 2017-11-05 21:59:13.561053 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(pgmap 56600067..56600783) refresh
 -1925> 2017-11-05 21:59:13.561094 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mdsmap 1..1) refresh
 -1924> 2017-11-05 21:59:13.561139 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(osdmap 503..1105) refresh
 -1923> 2017-11-05 21:59:13.561172 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(logm 55596915..55597634) refresh
 -1922> 2017-11-05 21:59:13.561179 7f23302e4700 10 mon.ceph03@2(peon).log v55597634 update_from_paxos
 -1921> 2017-11-05 21:59:13.561181 7f23302e4700 10 mon.ceph03@2(peon).log v55597634 update_from_paxos version 55597634 summary v 55597634
 -1920> 2017-11-05 21:59:13.561220 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(monmap 1..14) refresh
 -1919> 2017-11-05 21:59:13.561250 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(auth 26252..26331) refresh
 -1918> 2017-11-05 21:59:13.561257 7f23302e4700 10 mon.ceph03@2(peon).auth v26331 update_from_paxos
 -1917> 2017-11-05 21:59:13.561295 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgr 1..394) refresh
 -1916> 2017-11-05 21:59:13.561308 7f23302e4700  4 mgrc handle_mgr_map Got map version 394
 -1915> 2017-11-05 21:59:13.561311 7f23302e4700  4 mgrc handle_mgr_map Active mgr is now 10.10.1.21:6806/22556
 -1914> 2017-11-05 21:59:13.561349 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgrstat 56609513..56609633) refresh
 -1913> 2017-11-05 21:59:13.561355 7f23302e4700 10 mon.ceph03@2(peon).mgrstat  56609633
 -1912> 2017-11-05 21:59:13.561397 7f23302e4700 10 mon.ceph03@2(peon).mgrstat update_from_paxos v56609633 service_map e1
 -1911> 2017-11-05 21:59:13.561405 7f23302e4700 10 mon.ceph03@2(peon).mgrstat check_subs
 -1910> 2017-11-05 21:59:13.561407 7f23302e4700 20 mon.ceph03@2(peon).mgrstat update_logger
 -1909> 2017-11-05 21:59:13.561436 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(health 1..23) refresh
 -1908> 2017-11-05 21:59:13.561443 7f23302e4700 10 mon.ceph03@2(peon).health update_from_paxos
 -1907> 2017-11-05 21:59:13.561462 7f23302e4700 20 mon.ceph03@2(peon).health dump:{
    "quorum_health": {},
    "leader_health": {
        "MON_DOWN": {
            "severity": "HEALTH_WARN",
            "summary": {
                "message": "1/3 mons down, quorum ceph01mon,ceph02mon"
            },
            "detail": [
                {
                    "message": "mon.ceph03 (rank 2) addr 10.10.1.23:6789/0 is down (out of quorum)"
                }
            ]
        }
    }
}

 -1906> 2017-11-05 21:59:13.561489 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(pgmap 56600067..56600783) post_refresh
 -1905> 2017-11-05 21:59:13.561491 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mdsmap 1..1) post_refresh
 -1904> 2017-11-05 21:59:13.561493 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(osdmap 503..1105) post_refresh
 -1903> 2017-11-05 21:59:13.561495 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(logm 55596915..55597634) post_refresh
 -1902> 2017-11-05 21:59:13.561496 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(monmap 1..14) post_refresh
 -1901> 2017-11-05 21:59:13.561498 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(auth 26252..26331) post_refresh
 -1900> 2017-11-05 21:59:13.561499 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgr 1..394) post_refresh
 -1899> 2017-11-05 21:59:13.561501 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgrstat 56609513..56609633) post_refresh
 -1898> 2017-11-05 21:59:13.561502 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(health 1..23) post_refresh
 -1897> 2017-11-05 21:59:13.561579 7f23302e4700  1 -- 10.10.1.23:6789/0 <== mon.0 10.10.1.21:6789/0 248515235 ==== paxos(lease lc 112907739 fc 112907080 pn 0 opn 0) v4 ==== 84+0+0 (2163361469 0 0) 0x7f2342969b00 con 0x7f2342dd4800
 -1896> 2017-11-05 21:59:13.561597 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
 -1895> 2017-11-05 21:59:13.561606 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
 -1894> 2017-11-05 21:59:13.561609 7f23302e4700 20 is_capable service=mon command= read on cap allow *
 -1893> 2017-11-05 21:59:13.561611 7f23302e4700 20  allow so far , doing grant allow *
 -1892> 2017-11-05 21:59:13.561612 7f23302e4700 20  allow all
 -1891> 2017-11-05 21:59:13.561615 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
 -1890> 2017-11-05 21:59:13.561616 7f23302e4700 20  allow so far , doing grant allow *
 -1889> 2017-11-05 21:59:13.561617 7f23302e4700 20  allow all
 -1888> 2017-11-05 21:59:13.561622 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907739) handle_lease on 112907739 now 2017-11-05 21:59:18.516972
 -1887> 2017-11-05 21:59:13.561634 7f23302e4700  1 -- 10.10.1.23:6789/0 --> 10.10.1.21:6789/0 -- paxos(lease_ack lc 112907739 fc 112907080 pn 0 opn 0) v4 -- 0x7f2342969500 con 0
 -1886> 2017-11-05 21:59:13.561650 7f23302e4700 20 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907739) reset_lease_timeout - setting timeout event
 -1885> 2017-11-05 21:59:13.561656 7f23302e4700  5 mon.ceph03@2(peon).monmap v14 apply_mon_features wait for service to be writeable
 -1884> 2017-11-05 21:59:13.561660 7f23302e4700  5 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907739) is_readable = 1 - now=2017-11-05 21:59:13.561660 lease_expire=2017-11-05 21:59:18.516972 has v0 lc 112907739
 -1883> 2017-11-05 21:59:13.561722 7f232c2dc700 10 _calc_signature seq 12 front_crc_ = 2557579026 middle_crc = 0 data_crc = 0 sig = 17107270021298545528
 -1882> 2017-11-05 21:59:13.561742 7f232c2dc700 20 Putting signature in client message(seq # 12): sig = 17107270021298545528
 -1881> 2017-11-05 21:59:13.706944 7f232c2dc700 10 _calc_signature seq 248515236 front_crc_ = 2950603876 middle_crc = 0 data_crc = 0 sig = 14536828753144213213
 -1880> 2017-11-05 21:59:13.706962 7f232c2dc700  5 -- 10.10.1.23:6789/0 >> 10.10.1.21:6789/0 conn(0x7f2342dd4800 :-1 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=1124301 cs=1 l=0). rx mon.0 seq 248515236 0x7f2342969500 paxos(begin lc 112907739 fc 0 pn 13700 opn 0) v4
 -1879> 2017-11-05 21:59:13.707005 7f23302e4700  1 -- 10.10.1.23:6789/0 <== mon.0 10.10.1.21:6789/0 248515236 ==== paxos(begin lc 112907739 fc 0 pn 13700 opn 0) v4 ==== 3360+0+0 (2950603876 0 0) 0x7f2342969500 con 0x7f2342dd4800
 -1878> 2017-11-05 21:59:13.707035 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
 -1877> 2017-11-05 21:59:13.707045 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
 -1876> 2017-11-05 21:59:13.707048 7f23302e4700 20 is_capable service=mon command= read on cap allow *
 -1875> 2017-11-05 21:59:13.707051 7f23302e4700 20  allow so far , doing grant allow *
 -1874> 2017-11-05 21:59:13.707052 7f23302e4700 20  allow all
 -1873> 2017-11-05 21:59:13.707054 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
 -1872> 2017-11-05 21:59:13.707056 7f23302e4700 20  allow so far , doing grant allow *
 -1871> 2017-11-05 21:59:13.707062 7f23302e4700 20  allow all
 -1870> 2017-11-05 21:59:13.707069 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907739) handle_begin paxos(begin lc 112907739 fc 0 pn 13700 opn 0) v4
 -1869> 2017-11-05 21:59:13.707074 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos updating c 112907080..112907739) accepting value for 112907740 pn 13700
 -1868> 2017-11-05 21:59:13.749220 7f23302e4700  1 -- 10.10.1.23:6789/0 --> 10.10.1.21:6789/0 -- paxos(accept lc 112907739 fc 0 pn 13700 opn 0) v4 -- 0x7f2342969b00 con 0
 -1867> 2017-11-05 21:59:13.749323 7f232c2dc700 10 _calc_signature seq 13 front_crc_ = 1277342878 middle_crc = 0 data_crc = 0 sig = 9029584966831276109
 -1866> 2017-11-05 21:59:13.749341 7f232c2dc700 20 Putting signature in client message(seq # 13): sig = 9029584966831276109
 -1865> 2017-11-05 21:59:13.785280 7f232c2dc700 10 _calc_signature seq 248515237 front_crc_ = 893193776 middle_crc = 0 data_crc = 0 sig = 2612573258296941357
 -1864> 2017-11-05 21:59:13.785296 7f232c2dc700  5 -- 10.10.1.23:6789/0 >> 10.10.1.21:6789/0 conn(0x7f2342dd4800 :-1 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=1124301 cs=1 l=0). rx mon.0 seq 248515237 0x7f2342969b00 paxos(commit lc 112907740 fc 0 pn 13700 opn 0) v4
 -1863> 2017-11-05 21:59:13.785347 7f23302e4700  1 -- 10.10.1.23:6789/0 <== mon.0 10.10.1.21:6789/0 248515237 ==== paxos(commit lc 112907740 fc 0 pn 13700 opn 0) v4 ==== 3360+0+0 (893193776 0 0) 0x7f2342969b00 con 0x7f2342dd4800
 -1862> 2017-11-05 21:59:13.785376 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
 -1861> 2017-11-05 21:59:13.785385 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
 -1860> 2017-11-05 21:59:13.785388 7f23302e4700 20 is_capable service=mon command= read on cap allow *
 -1859> 2017-11-05 21:59:13.785391 7f23302e4700 20  allow so far , doing grant allow *
 -1858> 2017-11-05 21:59:13.785393 7f23302e4700 20  allow all
 -1857> 2017-11-05 21:59:13.785394 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
 -1856> 2017-11-05 21:59:13.785396 7f23302e4700 20  allow so far , doing grant allow *
 -1855> 2017-11-05 21:59:13.785397 7f23302e4700 20  allow all
 -1854> 2017-11-05 21:59:13.785403 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos updating c 112907080..112907739) handle_commit on 112907740
 -1853> 2017-11-05 21:59:13.785408 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos updating c 112907080..112907740) store_state [112907740..112907740]
 -1852> 2017-11-05 21:59:13.785510 7f232c2dc700 10 _calc_signature seq 248515238 front_crc_ = 4167574024 middle_crc = 0 data_crc = 0 sig = 4809870230413750984
 -1851> 2017-11-05 21:59:13.785521 7f232c2dc700  5 -- 10.10.1.23:6789/0 >> 10.10.1.21:6789/0 conn(0x7f2342dd4800 :-1 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=1124301 cs=1 l=0). rx mon.0 seq 248515238 0x7f2342969800 paxos(lease lc 112907740 fc 112907080 pn 0 opn 0) v4
 -1850> 2017-11-05 21:59:13.818516 7f232c2dc700 10 _calc_signature seq 248515239 front_crc_ = 1733102379 middle_crc = 0 data_crc = 0 sig = 9363108003920362173
 -1849> 2017-11-05 21:59:13.818545 7f232c2dc700  5 -- 10.10.1.23:6789/0 >> 10.10.1.21:6789/0 conn(0x7f2342dd4800 :-1 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=1124301 cs=1 l=0). rx mon.0 seq 248515239 0x7f234296a400 paxos(begin lc 112907740 fc 0 pn 13700 opn 0) v4
 -1848> 2017-11-05 21:59:13.826915 7f23302e4700 10 mon.ceph03@2(peon) e14 refresh_from_paxos
 -1847> 2017-11-05 21:59:13.826944 7f23302e4700 10 mon.ceph03@2(peon) e14 refresh_from_paxos no cluster_fingerprint
 -1846> 2017-11-05 21:59:13.826972 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(pgmap 56600067..56600783) refresh
 -1845> 2017-11-05 21:59:13.827012 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mdsmap 1..1) refresh
 -1844> 2017-11-05 21:59:13.827051 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(osdmap 503..1105) refresh
 -1843> 2017-11-05 21:59:13.827082 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(logm 55596915..55597634) refresh
 -1842> 2017-11-05 21:59:13.827088 7f23302e4700 10 mon.ceph03@2(peon).log v55597634 update_from_paxos
 -1841> 2017-11-05 21:59:13.827091 7f23302e4700 10 mon.ceph03@2(peon).log v55597634 update_from_paxos version 55597634 summary v 55597634
 -1840> 2017-11-05 21:59:13.827124 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(monmap 1..14) refresh
 -1839> 2017-11-05 21:59:13.827151 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(auth 26252..26331) refresh
 -1838> 2017-11-05 21:59:13.827158 7f23302e4700 10 mon.ceph03@2(peon).auth v26331 update_from_paxos
 -1837> 2017-11-05 21:59:13.827192 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgr 1..394) refresh
 -1836> 2017-11-05 21:59:13.827201 7f23302e4700  4 mgrc handle_mgr_map Got map version 394
 -1835> 2017-11-05 21:59:13.827203 7f23302e4700  4 mgrc handle_mgr_map Active mgr is now 10.10.1.21:6806/22556
 -1834> 2017-11-05 21:59:13.827239 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgrstat 56609513..56609634) refresh
 -1833> 2017-11-05 21:59:13.827249 7f23302e4700 10 mon.ceph03@2(peon).mgrstat  56609634
 -1832> 2017-11-05 21:59:13.827288 7f23302e4700 10 mon.ceph03@2(peon).mgrstat update_from_paxos v56609634 service_map e1
 -1831> 2017-11-05 21:59:13.827293 7f23302e4700 10 mon.ceph03@2(peon).mgrstat check_subs
 -1830> 2017-11-05 21:59:13.827295 7f23302e4700 20 mon.ceph03@2(peon).mgrstat update_logger
 -1829> 2017-11-05 21:59:13.827322 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(health 1..23) refresh
 -1828> 2017-11-05 21:59:13.827328 7f23302e4700 10 mon.ceph03@2(peon).health update_from_paxos
 -1827> 2017-11-05 21:59:13.827349 7f23302e4700 20 mon.ceph03@2(peon).health dump:{
    "quorum_health": {},
    "leader_health": {
        "MON_DOWN": {
            "severity": "HEALTH_WARN",
            "summary": {
                "message": "1/3 mons down, quorum ceph01mon,ceph02mon"
            },
            "detail": [
                {
                    "message": "mon.ceph03 (rank 2) addr 10.10.1.23:6789/0 is down (out of quorum)"
                }
            ]
        }
    }
}

 -1826> 2017-11-05 21:59:13.827372 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(pgmap 56600067..56600783) post_refresh
 -1825> 2017-11-05 21:59:13.827375 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mdsmap 1..1) post_refresh
 -1824> 2017-11-05 21:59:13.827377 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(osdmap 503..1105) post_refresh
 -1823> 2017-11-05 21:59:13.827378 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(logm 55596915..55597634) post_refresh
 -1822> 2017-11-05 21:59:13.827380 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(monmap 1..14) post_refresh
 -1821> 2017-11-05 21:59:13.827381 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(auth 26252..26331) post_refresh
 -1820> 2017-11-05 21:59:13.827383 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgr 1..394) post_refresh
 -1819> 2017-11-05 21:59:13.827384 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgrstat 56609513..56609634) post_refresh
 -1818> 2017-11-05 21:59:13.827385 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(health 1..23) post_refresh
 -1817> 2017-11-05 21:59:13.827455 7f23302e4700  1 -- 10.10.1.23:6789/0 <== mon.0 10.10.1.21:6789/0 248515238 ==== paxos(lease lc 112907740 fc 112907080 pn 0 opn 0) v4 ==== 84+0+0 (4167574024 0 0) 0x7f2342969800 con 0x7f2342dd4800
 -1816> 2017-11-05 21:59:13.827472 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
 -1815> 2017-11-05 21:59:13.827479 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
 -1814> 2017-11-05 21:59:13.827482 7f23302e4700 20 is_capable service=mon command= read on cap allow *
 -1813> 2017-11-05 21:59:13.827484 7f23302e4700 20  allow so far , doing grant allow *
 -1812> 2017-11-05 21:59:13.827485 7f23302e4700 20  allow all
 -1811> 2017-11-05 21:59:13.827487 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
 -1810> 2017-11-05 21:59:13.827489 7f23302e4700 20  allow so far , doing grant allow *
 -1809> 2017-11-05 21:59:13.827490 7f23302e4700 20  allow all
 -1808> 2017-11-05 21:59:13.827494 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907740) handle_lease on 112907740 now 2017-11-05 21:59:18.783147
 -1807> 2017-11-05 21:59:13.827509 7f23302e4700  1 -- 10.10.1.23:6789/0 --> 10.10.1.21:6789/0 -- paxos(lease_ack lc 112907740 fc 112907080 pn 0 opn 0) v4 -- 0x7f2342969b00 con 0
 -1806> 2017-11-05 21:59:13.827525 7f23302e4700 20 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907740) reset_lease_timeout - setting timeout event
 -1805> 2017-11-05 21:59:13.827532 7f23302e4700  5 mon.ceph03@2(peon).monmap v14 apply_mon_features wait for service to be writeable
 -1804> 2017-11-05 21:59:13.827536 7f23302e4700  5 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907740) is_readable = 1 - now=2017-11-05 21:59:13.827537 lease_expire=2017-11-05 21:59:18.783147 has v0 lc 112907740
 -1803> 2017-11-05 21:59:13.827554 7f23302e4700  1 -- 10.10.1.23:6789/0 <== mon.0 10.10.1.21:6789/0 248515239 ==== paxos(begin lc 112907740 fc 0 pn 13700 opn 0) v4 ==== 11421+0+0 (1733102379 0 0) 0x7f234296a400 con 0x7f2342dd4800
 -1802> 2017-11-05 21:59:13.827563 7f232c2dc700 10 _calc_signature seq 14 front_crc_ = 1046939931 middle_crc = 0 data_crc = 0 sig = 137334039882502561
 -1801> 2017-11-05 21:59:13.827583 7f232c2dc700 20 Putting signature in client message(seq # 14): sig = 137334039882502561
 -1800> 2017-11-05 21:59:13.827566 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
 -1799> 2017-11-05 21:59:13.827590 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
 -1798> 2017-11-05 21:59:13.827601 7f23302e4700 20 is_capable service=mon command= read on cap allow *
 -1797> 2017-11-05 21:59:13.827603 7f23302e4700 20  allow so far , doing grant allow *
 -1796> 2017-11-05 21:59:13.827606 7f23302e4700 20  allow all
 -1795> 2017-11-05 21:59:13.827607 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
 -1794> 2017-11-05 21:59:13.827609 7f23302e4700 20  allow so far , doing grant allow *
 -1793> 2017-11-05 21:59:13.827610 7f23302e4700 20  allow all
 -1792> 2017-11-05 21:59:13.827619 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907740) handle_begin paxos(begin lc 112907740 fc 0 pn 13700 opn 0) v4
 -1791> 2017-11-05 21:59:13.827623 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos updating c 112907080..112907740) accepting value for 112907741 pn 13700
 -1790> 2017-11-05 21:59:13.871272 7f23302e4700  1 -- 10.10.1.23:6789/0 --> 10.10.1.21:6789/0 -- paxos(accept lc 112907740 fc 0 pn 13700 opn 0) v4 -- 0x7f2342969800 con 0
 -1789> 2017-11-05 21:59:13.871338 7f232c2dc700 10 _calc_signature seq 15 front_crc_ = 3677845416 middle_crc = 0 data_crc = 0 sig = 4007921578359993347
 -1788> 2017-11-05 21:59:13.871352 7f232c2dc700 20 Putting signature in client message(seq # 15): sig = 4007921578359993347
 -1787> 2017-11-05 21:59:13.907279 7f232c2dc700 10 _calc_signature seq 248515240 front_crc_ = 2867548840 middle_crc = 0 data_crc = 0 sig = 15577785751050191599
 -1786> 2017-11-05 21:59:13.907300 7f232c2dc700  5 -- 10.10.1.23:6789/0 >> 10.10.1.21:6789/0 conn(0x7f2342dd4800 :-1 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=1124301 cs=1 l=0). rx mon.0 seq 248515240 0x7f2342969800 paxos(commit lc 112907741 fc 0 pn 13700 opn 0) v4
 -1785> 2017-11-05 21:59:13.907348 7f23302e4700  1 -- 10.10.1.23:6789/0 <== mon.0 10.10.1.21:6789/0 248515240 ==== paxos(commit lc 112907741 fc 0 pn 13700 opn 0) v4 ==== 11421+0+0 (2867548840 0 0) 0x7f2342969800 con 0x7f2342dd4800
 -1784> 2017-11-05 21:59:13.907373 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
 -1783> 2017-11-05 21:59:13.907381 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
 -1782> 2017-11-05 21:59:13.907384 7f23302e4700 20 is_capable service=mon command= read on cap allow *
 -1781> 2017-11-05 21:59:13.907388 7f23302e4700 20  allow so far , doing grant allow *
 -1780> 2017-11-05 21:59:13.907390 7f23302e4700 20  allow all
 -1779> 2017-11-05 21:59:13.907391 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
 -1778> 2017-11-05 21:59:13.907392 7f23302e4700 20  allow so far , doing grant allow *
 -1777> 2017-11-05 21:59:13.907393 7f23302e4700 20  allow all
 -1776> 2017-11-05 21:59:13.907401 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos updating c 112907080..112907740) handle_commit on 112907741
 -1775> 2017-11-05 21:59:13.907405 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos updating c 112907080..112907741) store_state [112907741..112907741]
 -1774> 2017-11-05 21:59:13.907657 7f232c2dc700 10 _calc_signature seq 248515241 front_crc_ = 3270755471 middle_crc = 0 data_crc = 0 sig = 3345080007523614143
 -1773> 2017-11-05 21:59:13.907667 7f232c2dc700  5 -- 10.10.1.23:6789/0 >> 10.10.1.21:6789/0 conn(0x7f2342dd4800 :-1 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=1124301 cs=1 l=0). rx mon.0 seq 248515241 0x7f2342969b00 paxos(lease lc 112907741 fc 112907080 pn 0 opn 0) v4
 -1772> 2017-11-05 21:59:13.907848 7f232c2dc700 10 _calc_signature seq 248515242 front_crc_ = 1530611789 middle_crc = 0 data_crc = 0 sig = 10373082544299377680
 -1771> 2017-11-05 21:59:13.907860 7f232c2dc700  5 -- 10.10.1.23:6789/0 >> 10.10.1.21:6789/0 conn(0x7f2342dd4800 :-1 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=1124301 cs=1 l=0). rx mon.0 seq 248515242 0x7f234298dd40 route(log(last 1) v1 tid 1) v3
 -1770> 2017-11-05 21:59:13.949109 7f23302e4700 10 mon.ceph03@2(peon) e14 refresh_from_paxos
 -1769> 2017-11-05 21:59:13.949140 7f23302e4700 10 mon.ceph03@2(peon) e14 refresh_from_paxos no cluster_fingerprint
 -1768> 2017-11-05 21:59:13.949169 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(pgmap 56600067..56600783) refresh
 -1767> 2017-11-05 21:59:13.949206 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mdsmap 1..1) refresh
 -1766> 2017-11-05 21:59:13.949256 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(osdmap 503..1105) refresh
 -1765> 2017-11-05 21:59:13.949285 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(logm 55596915..55597635) refresh
 -1764> 2017-11-05 21:59:13.949290 7f23302e4700 10 mon.ceph03@2(peon).log v55597635 update_from_paxos
 -1763> 2017-11-05 21:59:13.949293 7f23302e4700 10 mon.ceph03@2(peon).log v55597635 update_from_paxos version 55597635 summary v 55597634
 -1762> 2017-11-05 21:59:13.949300 7f23302e4700 10 mon.ceph03@2(peon).log v55597635 update_from_paxos latest full 55597634
 -1761> 2017-11-05 21:59:13.949315 7f23302e4700  7 mon.ceph03@2(peon).log v55597635 update_from_paxos applying incremental log 55597635 2017-11-05 21:59:12.841468 mon.ceph03 mon.2 10.10.1.23:6789/0 1 : cluster [INF] mon.ceph03 calling new monitor election
 -1760> 2017-11-05 21:59:13.949335 7f23302e4700 20 mon.ceph03@2(peon).log v55597635 update_from_paxos logging for channel 'cluster' to file '/var/log/ceph/ceph.log'
 -1759> 2017-11-05 21:59:13.949351 7f23302e4700  7 mon.ceph03@2(peon).log v55597635 update_from_paxos applying incremental log 55597635 2017-11-05 21:59:12.884285 mon.ceph01mon mon.0 10.10.1.21:6789/0 855 : cluster [INF] mon.ceph01mon calling new monitor election
 -1758> 2017-11-05 21:59:13.949361 7f23302e4700 20 mon.ceph03@2(peon).log v55597635 update_from_paxos logging for channel 'cluster' to file '/var/log/ceph/ceph.log'
 -1757> 2017-11-05 21:59:13.949370 7f23302e4700  7 mon.ceph03@2(peon).log v55597635 update_from_paxos applying incremental log 55597635 2017-11-05 21:59:12.884938 mon.ceph02mon mon.1 10.10.1.22:6789/0 361 : cluster [INF] mon.ceph02mon calling new monitor election
 -1756> 2017-11-05 21:59:13.949376 7f23302e4700 20 mon.ceph03@2(peon).log v55597635 update_from_paxos logging for channel 'cluster' to file '/var/log/ceph/ceph.log'
 -1755> 2017-11-05 21:59:13.949384 7f23302e4700  7 mon.ceph03@2(peon).log v55597635 update_from_paxos applying incremental log 55597635 2017-11-05 21:59:12.984648 mon.ceph01mon mon.0 10.10.1.21:6789/0 856 : cluster [INF] mon.ceph01mon@0 won leader election with quorum 0,1,2
 -1754> 2017-11-05 21:59:13.949392 7f23302e4700 20 mon.ceph03@2(peon).log v55597635 update_from_paxos logging for channel 'cluster' to file '/var/log/ceph/ceph.log'
 -1753> 2017-11-05 21:59:13.949401 7f23302e4700  7 mon.ceph03@2(peon).log v55597635 update_from_paxos applying incremental log 55597635 2017-11-05 21:59:13.018039 mon.ceph01mon mon.0 10.10.1.21:6789/0 857 : cluster [WRN] overall HEALTH_WARN crush map has legacy tunables (require bobtail, min is firefly); nodeep-scrub flag(s) set; 1/3 mons down, quorum ceph01mon,ceph02mon
 -1752> 2017-11-05 21:59:13.949411 7f23302e4700 20 mon.ceph03@2(peon).log v55597635 update_from_paxos logging for channel 'cluster' to file '/var/log/ceph/ceph.log'
 -1751> 2017-11-05 21:59:13.949419 7f23302e4700  7 mon.ceph03@2(peon).log v55597635 update_from_paxos applying incremental log 55597635 2017-11-05 21:59:13.289046 mon.ceph01mon mon.0 10.10.1.21:6789/0 858 : cluster [INF] mon.1 10.10.1.22:6789/0
 -1750> 2017-11-05 21:59:13.949424 7f23302e4700 20 mon.ceph03@2(peon).log v55597635 update_from_paxos logging for channel 'cluster' to file '/var/log/ceph/ceph.log'
 -1749> 2017-11-05 21:59:13.949433 7f23302e4700  7 mon.ceph03@2(peon).log v55597635 update_from_paxos applying incremental log 55597635 2017-11-05 21:59:13.294021 mon.ceph01mon mon.0 10.10.1.21:6789/0 859 : cluster [INF] monmap e14: 3 mons at {ceph01mon=10.10.1.21:6789/0,ceph02mon=10.10.1.22:6789/0,ceph03=10.10.1.23:6789/0}
 -1748> 2017-11-05 21:59:13.949441 7f23302e4700 20 mon.ceph03@2(peon).log v55597635 update_from_paxos logging for channel 'cluster' to file '/var/log/ceph/ceph.log'
 -1747> 2017-11-05 21:59:13.949450 7f23302e4700  7 mon.ceph03@2(peon).log v55597635 update_from_paxos applying incremental log 55597635 2017-11-05 21:59:13.294068 mon.ceph01mon mon.0 10.10.1.21:6789/0 860 : cluster [DBG] fsmap
 -1746> 2017-11-05 21:59:13.949459 7f23302e4700 20 mon.ceph03@2(peon).log v55597635 update_from_paxos logging for channel 'cluster' to file '/var/log/ceph/ceph.log'
 -1745> 2017-11-05 21:59:13.949464 7f23302e4700  7 mon.ceph03@2(peon).log v55597635 update_from_paxos applying incremental log 55597635 2017-11-05 21:59:13.294143 mon.ceph01mon mon.0 10.10.1.21:6789/0 861 : cluster [DBG] osdmap e1105: 9 total, 9 up, 9 in
 -1744> 2017-11-05 21:59:13.949469 7f23302e4700 20 mon.ceph03@2(peon).log v55597635 update_from_paxos logging for channel 'cluster' to file '/var/log/ceph/ceph.log'
 -1743> 2017-11-05 21:59:13.949474 7f23302e4700  7 mon.ceph03@2(peon).log v55597635 update_from_paxos applying incremental log 55597635 2017-11-05 21:59:13.294231 mon.ceph01mon mon.0 10.10.1.21:6789/0 862 : cluster [DBG] mgrmap e394: ceph01mgr(active), standbys: ceph02mgr
 -1742> 2017-11-05 21:59:13.949479 7f23302e4700 20 mon.ceph03@2(peon).log v55597635 update_from_paxos logging for channel 'cluster' to file '/var/log/ceph/ceph.log'
 -1741> 2017-11-05 21:59:13.949486 7f23302e4700  7 mon.ceph03@2(peon).log v55597635 update_from_paxos applying incremental log 55597635 2017-11-05 21:59:13.438995 mon.ceph01mon mon.0 10.10.1.21:6789/0 863 : cluster [INF] mon.2 10.10.1.23:6789/0
 -1740> 2017-11-05 21:59:13.949494 7f23302e4700 20 mon.ceph03@2(peon).log v55597635 update_from_paxos logging for channel 'cluster' to file '/var/log/ceph/ceph.log'
 -1739> 2017-11-05 21:59:13.949513 7f23302e4700 15 mon.ceph03@2(peon).log v55597635 update_from_paxos logging for 1 channels
 -1738> 2017-11-05 21:59:13.949518 7f23302e4700 15 mon.ceph03@2(peon).log v55597635 update_from_paxos channel 'cluster' logging 1163 bytes
 -1737> 2017-11-05 21:59:13.949568 7f23302e4700 10 mon.ceph03@2(peon).log v55597635 check_subs
 -1736> 2017-11-05 21:59:13.949636 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(monmap 1..14) refresh
 -1735> 2017-11-05 21:59:13.949663 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(auth 26252..26331) refresh
 -1734> 2017-11-05 21:59:13.949669 7f23302e4700 10 mon.ceph03@2(peon).auth v26331 update_from_paxos
 -1733> 2017-11-05 21:59:13.949705 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgr 1..394) refresh
 -1732> 2017-11-05 21:59:13.949716 7f23302e4700  4 mgrc handle_mgr_map Got map version 394
 -1731> 2017-11-05 21:59:13.949718 7f23302e4700  4 mgrc handle_mgr_map Active mgr is now 10.10.1.21:6806/22556
 -1730> 2017-11-05 21:59:13.949750 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgrstat 56609513..56609634) refresh
 -1729> 2017-11-05 21:59:13.949756 7f23302e4700 10 mon.ceph03@2(peon).mgrstat  56609634
 -1728> 2017-11-05 21:59:13.949791 7f23302e4700 10 mon.ceph03@2(peon).mgrstat update_from_paxos v56609634 service_map e1
 -1727> 2017-11-05 21:59:13.949796 7f23302e4700 10 mon.ceph03@2(peon).mgrstat check_subs
 -1726> 2017-11-05 21:59:13.949797 7f23302e4700 20 mon.ceph03@2(peon).mgrstat update_logger
 -1725> 2017-11-05 21:59:13.949822 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(health 1..23) refresh
 -1724> 2017-11-05 21:59:13.949828 7f23302e4700 10 mon.ceph03@2(peon).health update_from_paxos
 -1723> 2017-11-05 21:59:13.949849 7f23302e4700 20 mon.ceph03@2(peon).health dump:{
    "quorum_health": {},
    "leader_health": {
        "MON_DOWN": {
            "severity": "HEALTH_WARN",
            "summary": {
                "message": "1/3 mons down, quorum ceph01mon,ceph02mon"
            },
            "detail": [
                {
                    "message": "mon.ceph03 (rank 2) addr 10.10.1.23:6789/0 is down (out of quorum)"
                }
            ]
        }
    }
}

 -1722> 2017-11-05 21:59:13.949874 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(pgmap 56600067..56600783) post_refresh
 -1721> 2017-11-05 21:59:13.949876 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mdsmap 1..1) post_refresh
 -1720> 2017-11-05 21:59:13.949878 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(osdmap 503..1105) post_refresh
 -1719> 2017-11-05 21:59:13.949879 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(logm 55596915..55597635) post_refresh
 -1718> 2017-11-05 21:59:13.949881 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(monmap 1..14) post_refresh
 -1717> 2017-11-05 21:59:13.949882 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(auth 26252..26331) post_refresh
 -1716> 2017-11-05 21:59:13.949883 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgr 1..394) post_refresh
 -1715> 2017-11-05 21:59:13.949885 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgrstat 56609513..56609634) post_refresh
 -1714> 2017-11-05 21:59:13.949886 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(health 1..23) post_refresh
 -1713> 2017-11-05 21:59:13.949945 7f23302e4700  1 -- 10.10.1.23:6789/0 <== mon.0 10.10.1.21:6789/0 248515241 ==== paxos(lease lc 112907741 fc 112907080 pn 0 opn 0) v4 ==== 84+0+0 (3270755471 0 0) 0x7f2342969b00 con 0x7f2342dd4800
 -1712> 2017-11-05 21:59:13.949961 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
 -1711> 2017-11-05 21:59:13.949968 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
 -1710> 2017-11-05 21:59:13.949971 7f23302e4700 20 is_capable service=mon command= read on cap allow *
 -1709> 2017-11-05 21:59:13.949973 7f23302e4700 20  allow so far , doing grant allow *
 -1708> 2017-11-05 21:59:13.949974 7f23302e4700 20  allow all
 -1707> 2017-11-05 21:59:13.949975 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
 -1706> 2017-11-05 21:59:13.949979 7f23302e4700 20  allow so far , doing grant allow *
 -1705> 2017-11-05 21:59:13.949981 7f23302e4700 20  allow all
 -1704> 2017-11-05 21:59:13.949984 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907741) handle_lease on 112907741 now 2017-11-05 21:59:18.905672
 -1703> 2017-11-05 21:59:13.949994 7f23302e4700  1 -- 10.10.1.23:6789/0 --> 10.10.1.21:6789/0 -- paxos(lease_ack lc 112907741 fc 112907080 pn 0 opn 0) v4 -- 0x7f2342969800 con 0
 -1702> 2017-11-05 21:59:13.950008 7f23302e4700 20 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907741) reset_lease_timeout - setting timeout event
 -1701> 2017-11-05 21:59:13.950014 7f23302e4700  5 mon.ceph03@2(peon).monmap v14 apply_mon_features wait for service to be writeable
 -1700> 2017-11-05 21:59:13.950017 7f23302e4700  5 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907741) is_readable = 1 - now=2017-11-05 21:59:13.950017 lease_expire=2017-11-05 21:59:18.905672 has v0 lc 112907741
 -1699> 2017-11-05 21:59:13.950029 7f23302e4700  1 -- 10.10.1.23:6789/0 <== mon.0 10.10.1.21:6789/0 248515242 ==== route(log(last 1) v1 tid 1) v3 ==== 175+0+0 (1530611789 0 0) 0x7f234298dd40 con 0x7f2342dd4800
 -1698> 2017-11-05 21:59:13.950040 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
 -1697> 2017-11-05 21:59:13.950045 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
 -1696> 2017-11-05 21:59:13.950047 7f23302e4700 20 is_capable service=mon command= read on cap allow *
 -1695> 2017-11-05 21:59:13.950048 7f23302e4700 20  allow so far , doing grant allow *
 -1694> 2017-11-05 21:59:13.950050 7f23302e4700 20  allow all
 -1693> 2017-11-05 21:59:13.950053 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
 -1692> 2017-11-05 21:59:13.950054 7f23302e4700 20  allow so far , doing grant allow *
 -1691> 2017-11-05 21:59:13.950055 7f23302e4700 20  allow all
 -1690> 2017-11-05 21:59:13.950056 7f23302e4700 10 mon.ceph03@2(peon) e14 handle_route log(last 1) v1 to unknown.0 -
 -1689> 2017-11-05 21:59:13.950052 7f232c2dc700 10 _calc_signature seq 16 front_crc_ = 847570628 middle_crc = 0 data_crc = 0 sig = 12628439953592722512
 -1688> 2017-11-05 21:59:13.950062 7f232c2dc700 20 Putting signature in client message(seq # 16): sig = 12628439953592722512
 -1687> 2017-11-05 21:59:13.950060 7f23302e4700  1 -- 10.10.1.23:6789/0 --> 10.10.1.23:6789/0 -- log(last 1) v1 -- 0x7f2342957800 con 0
 -1686> 2017-11-05 21:59:13.950098 7f23302e4700  1 -- 10.10.1.23:6789/0 <== mon.2 10.10.1.23:6789/0 0 ==== log(last 1) v1 ==== 0+0+0 (2876517680 0 0) 0x7f2342957800 con 0x7f23431d6000
 -1685> 2017-11-05 21:59:13.950109 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930a80 for mon.2 10.10.1.23:6789/0
 -1684> 2017-11-05 21:59:13.950116 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
 -1683> 2017-11-05 21:59:13.950118 7f23302e4700 20 is_capable service=mon command= read on cap allow *
 -1682> 2017-11-05 21:59:13.950119 7f23302e4700 20  allow so far , doing grant allow *
 -1681> 2017-11-05 21:59:13.950120 7f23302e4700 20  allow all
 -1680> 2017-11-05 21:59:13.950122 7f23302e4700 10 log_client handle_log_ack log(last 1) v1
 -1679> 2017-11-05 21:59:13.950124 7f23302e4700 10 log_client  logged 2017-11-05 21:59:12.841468 mon.ceph03 mon.2 10.10.1.23:6789/0 1 : cluster [INF] mon.ceph03 calling new monitor election
 -1678> 2017-11-05 21:59:14.937043 7f232c2dc700 10 _calc_signature seq 248515243 front_crc_ = 1914412222 middle_crc = 0 data_crc = 0 sig = 16176292074621219325
 -1677> 2017-11-05 21:59:14.937065 7f232c2dc700  5 -- 10.10.1.23:6789/0 >> 10.10.1.21:6789/0 conn(0x7f2342dd4800 :-1 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=1124301 cs=1 l=0). rx mon.0 seq 248515243 0x7f2342969800 paxos(begin lc 112907741 fc 0 pn 13700 opn 0) v4
 -1676> 2017-11-05 21:59:14.937135 7f23302e4700  1 -- 10.10.1.23:6789/0 <== mon.0 10.10.1.21:6789/0 248515243 ==== paxos(begin lc 112907741 fc 0 pn 13700 opn 0) v4 ==== 3360+0+0 (1914412222 0 0) 0x7f2342969800 con 0x7f2342dd4800
 -1675> 2017-11-05 21:59:14.937173 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
 -1674> 2017-11-05 21:59:14.937184 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
 -1673> 2017-11-05 21:59:14.937188 7f23302e4700 20 is_capable service=mon command= read on cap allow *
 -1672> 2017-11-05 21:59:14.937192 7f23302e4700 20  allow so far , doing grant allow *
 -1671> 2017-11-05 21:59:14.937193 7f23302e4700 20  allow all
 -1670> 2017-11-05 21:59:14.937195 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
 -1669> 2017-11-05 21:59:14.937210 7f23302e4700 20  allow so far , doing grant allow *
 -1668> 2017-11-05 21:59:14.937211 7f23302e4700 20  allow all
 -1667> 2017-11-05 21:59:14.937217 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907741) handle_begin paxos(begin lc 112907741 fc 0 pn 13700 opn 0) v4
 -1666> 2017-11-05 21:59:14.937222 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos updating c 112907080..112907741) accepting value for 112907742 pn 13700
 -1665> 2017-11-05 21:59:14.979029 7f23302e4700  1 -- 10.10.1.23:6789/0 --> 10.10.1.21:6789/0 -- paxos(accept lc 112907741 fc 0 pn 13700 opn 0) v4 -- 0x7f2342969b00 con 0
 -1664> 2017-11-05 21:59:14.979132 7f232c2dc700 10 _calc_signature seq 17 front_crc_ = 87764062 middle_crc = 0 data_crc = 0 sig = 8597943680905522514
 -1663> 2017-11-05 21:59:14.979149 7f232c2dc700 20 Putting signature in client message(seq # 17): sig = 8597943680905522514
 -1662> 2017-11-05 21:59:15.014726 7f232c2dc700 10 _calc_signature seq 248515244 front_crc_ = 3809903813 middle_crc = 0 data_crc = 0 sig = 6400586654888549434
 -1661> 2017-11-05 21:59:15.014742 7f232c2dc700  5 -- 10.10.1.23:6789/0 >> 10.10.1.21:6789/0 conn(0x7f2342dd4800 :-1 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=1124301 cs=1 l=0). rx mon.0 seq 248515244 0x7f2342969b00 paxos(commit lc 112907742 fc 0 pn 13700 opn 0) v4
 -1660> 2017-11-05 21:59:15.014806 7f23302e4700  1 -- 10.10.1.23:6789/0 <== mon.0 10.10.1.21:6789/0 248515244 ==== paxos(commit lc 112907742 fc 0 pn 13700 opn 0) v4 ==== 3360+0+0 (3809903813 0 0) 0x7f2342969b00 con 0x7f2342dd4800
 -1659> 2017-11-05 21:59:15.014840 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
 -1658> 2017-11-05 21:59:15.014860 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
 -1657> 2017-11-05 21:59:15.014863 7f23302e4700 20 is_capable service=mon command= read on cap allow *
 -1656> 2017-11-05 21:59:15.014866 7f23302e4700 20  allow so far , doing grant allow *
 -1655> 2017-11-05 21:59:15.014867 7f23302e4700 20  allow all
 -1654> 2017-11-05 21:59:15.014869 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
 -1653> 2017-11-05 21:59:15.014871 7f23302e4700 20  allow so far , doing grant allow *
 -1652> 2017-11-05 21:59:15.014872 7f23302e4700 20  allow all
 -1651> 2017-11-05 21:59:15.014877 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos updating c 112907080..112907741) handle_commit on 112907742
 -1650> 2017-11-05 21:59:15.014882 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos updating c 112907080..112907742) store_state [112907742..112907742]
 -1649> 2017-11-05 21:59:15.015008 7f232c2dc700 10 _calc_signature seq 248515245 front_crc_ = 561089878 middle_crc = 0 data_crc = 0 sig = 17226791919200925804
 -1648> 2017-11-05 21:59:15.015020 7f232c2dc700  5 -- 10.10.1.23:6789/0 >> 10.10.1.21:6789/0 conn(0x7f2342dd4800 :-1 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=1124301 cs=1 l=0). rx mon.0 seq 248515245 0x7f234296a100 paxos(lease lc 112907742 fc 112907080 pn 0 opn 0) v4
 -1647> 2017-11-05 21:59:15.056780 7f23302e4700 10 mon.ceph03@2(peon) e14 refresh_from_paxos
 -1646> 2017-11-05 21:59:15.056816 7f23302e4700 10 mon.ceph03@2(peon) e14 refresh_from_paxos no cluster_fingerprint
 -1645> 2017-11-05 21:59:15.056852 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(pgmap 56600067..56600783) refresh
 -1644> 2017-11-05 21:59:15.056896 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mdsmap 1..1) refresh
 -1643> 2017-11-05 21:59:15.056944 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(osdmap 503..1105) refresh
 -1642> 2017-11-05 21:59:15.056978 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(logm 55596915..55597635) refresh
 -1641> 2017-11-05 21:59:15.056984 7f23302e4700 10 mon.ceph03@2(peon).log v55597635 update_from_paxos
 -1640> 2017-11-05 21:59:15.056987 7f23302e4700 10 mon.ceph03@2(peon).log v55597635 update_from_paxos version 55597635 summary v 55597635
 -1639> 2017-11-05 21:59:15.057031 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(monmap 1..14) refresh
 -1638> 2017-11-05 21:59:15.057063 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(auth 26252..26331) refresh
 -1637> 2017-11-05 21:59:15.057069 7f23302e4700 10 mon.ceph03@2(peon).auth v26331 update_from_paxos
 -1636> 2017-11-05 21:59:15.057111 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgr 1..394) refresh
 -1635> 2017-11-05 21:59:15.057125 7f23302e4700  4 mgrc handle_mgr_map Got map version 394
 -1634> 2017-11-05 21:59:15.057134 7f23302e4700  4 mgrc handle_mgr_map Active mgr is now 10.10.1.21:6806/22556
 -1633> 2017-11-05 21:59:15.057175 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgrstat 56609513..56609635) refresh
 -1632> 2017-11-05 21:59:15.057183 7f23302e4700 10 mon.ceph03@2(peon).mgrstat  56609635
 -1631> 2017-11-05 21:59:15.057227 7f23302e4700 10 mon.ceph03@2(peon).mgrstat update_from_paxos v56609635 service_map e1
 -1630> 2017-11-05 21:59:15.057233 7f23302e4700 10 mon.ceph03@2(peon).mgrstat check_subs
 -1629> 2017-11-05 21:59:15.057235 7f23302e4700 20 mon.ceph03@2(peon).mgrstat update_logger
 -1628> 2017-11-05 21:59:15.057268 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(health 1..23) refresh
 -1627> 2017-11-05 21:59:15.057274 7f23302e4700 10 mon.ceph03@2(peon).health update_from_paxos
 -1626> 2017-11-05 21:59:15.057296 7f23302e4700 20 mon.ceph03@2(peon).health dump:{
    "quorum_health": {},
    "leader_health": {
        "MON_DOWN": {
            "severity": "HEALTH_WARN",
            "summary": {
                "message": "1/3 mons down, quorum ceph01mon,ceph02mon"
            },
            "detail": [
                {
                    "message": "mon.ceph03 (rank 2) addr 10.10.1.23:6789/0 is down (out of quorum)"
                }
            ]
        }
    }
}

 -1625> 2017-11-05 21:59:15.057322 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(pgmap 56600067..56600783) post_refresh
 -1624> 2017-11-05 21:59:15.057326 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mdsmap 1..1) post_refresh
 -1623> 2017-11-05 21:59:15.057328 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(osdmap 503..1105) post_refresh
 -1622> 2017-11-05 21:59:15.057330 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(logm 55596915..55597635) post_refresh
 -1621> 2017-11-05 21:59:15.057332 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(monmap 1..14) post_refresh
 -1620> 2017-11-05 21:59:15.057334 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(auth 26252..26331) post_refresh
 -1619> 2017-11-05 21:59:15.057335 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgr 1..394) post_refresh
 -1618> 2017-11-05 21:59:15.057337 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgrstat 56609513..56609635) post_refresh
 -1617> 2017-11-05 21:59:15.057339 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(health 1..23) post_refresh
 -1616> 2017-11-05 21:59:15.057409 7f23302e4700  1 -- 10.10.1.23:6789/0 <== mon.0 10.10.1.21:6789/0 248515245 ==== paxos(lease lc 112907742 fc 112907080 pn 0 opn 0) v4 ==== 84+0+0 (561089878 0 0) 0x7f234296a100 con 0x7f2342dd4800
 -1615> 2017-11-05 21:59:15.057427 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
 -1614> 2017-11-05 21:59:15.057435 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
 -1613> 2017-11-05 21:59:15.057438 7f23302e4700 20 is_capable service=mon command= read on cap allow *
 -1612> 2017-11-05 21:59:15.057441 7f23302e4700 20  allow so far , doing grant allow *
 -1611> 2017-11-05 21:59:15.057442 7f23302e4700 20  allow all
 -1610> 2017-11-05 21:59:15.057444 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
 -1609> 2017-11-05 21:59:15.057445 7f23302e4700 20  allow so far , doing grant allow *
 -1608> 2017-11-05 21:59:15.057447 7f23302e4700 20  allow all
 -1607> 2017-11-05 21:59:15.057451 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907742) handle_lease on 112907742 now 2017-11-05 21:59:20.013155
 -1606> 2017-11-05 21:59:15.057464 7f23302e4700  1 -- 10.10.1.23:6789/0 --> 10.10.1.21:6789/0 -- paxos(lease_ack lc 112907742 fc 112907080 pn 0 opn 0) v4 -- 0x7f2342969b00 con 0
 -1605> 2017-11-05 21:59:15.057483 7f23302e4700 20 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907742) reset_lease_timeout - setting timeout event
 -1604> 2017-11-05 21:59:15.057490 7f23302e4700  5 mon.ceph03@2(peon).monmap v14 apply_mon_features wait for service to be writeable
 -1603> 2017-11-05 21:59:15.057494 7f23302e4700  5 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907742) is_readable = 1 - now=2017-11-05 21:59:15.057494 lease_expire=2017-11-05 21:59:20.013155 has v0 lc 112907742
 -1602> 2017-11-05 21:59:15.057553 7f232c2dc700 10 _calc_signature seq 18 front_crc_ = 3194447853 middle_crc = 0 data_crc = 0 sig = 17594015652951604420
 -1601> 2017-11-05 21:59:15.057571 7f232c2dc700 20 Putting signature in client message(seq # 18): sig = 17594015652951604420
 -1600> 2017-11-05 21:59:16.254989 7f232c2dc700 10 _calc_signature seq 248515246 front_crc_ = 3080749871 middle_crc = 0 data_crc = 0 sig = 13444557980616518344
 -1599> 2017-11-05 21:59:16.255022 7f232c2dc700  5 -- 10.10.1.23:6789/0 >> 10.10.1.21:6789/0 conn(0x7f2342dd4800 :-1 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=1124301 cs=1 l=0). rx mon.0 seq 248515246 0x7f2342969b00 paxos(begin lc 112907742 fc 0 pn 13700 opn 0) v4
 -1598> 2017-11-05 21:59:16.255070 7f23302e4700  1 -- 10.10.1.23:6789/0 <== mon.0 10.10.1.21:6789/0 248515246 ==== paxos(begin lc 112907742 fc 0 pn 13700 opn 0) v4 ==== 3368+0+0 (3080749871 0 0) 0x7f2342969b00 con 0x7f2342dd4800
 -1597> 2017-11-05 21:59:16.255107 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
 -1596> 2017-11-05 21:59:16.255118 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
 -1595> 2017-11-05 21:59:16.255122 7f23302e4700 20 is_capable service=mon command= read on cap allow *
 -1594> 2017-11-05 21:59:16.255126 7f23302e4700 20  allow so far , doing grant allow *
 -1593> 2017-11-05 21:59:16.255128 7f23302e4700 20  allow all
 -1592> 2017-11-05 21:59:16.255132 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
 -1591> 2017-11-05 21:59:16.255134 7f23302e4700 20  allow so far , doing grant allow *
 -1590> 2017-11-05 21:59:16.255135 7f23302e4700 20  allow all
 -1589> 2017-11-05 21:59:16.255142 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907742) handle_begin paxos(begin lc 112907742 fc 0 pn 13700 opn 0) v4
 -1588> 2017-11-05 21:59:16.255147 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos updating c 112907080..112907742) accepting value for 112907743 pn 13700
 -1587> 2017-11-05 21:59:16.296729 7f23302e4700  1 -- 10.10.1.23:6789/0 --> 10.10.1.21:6789/0 -- paxos(accept lc 112907742 fc 0 pn 13700 opn 0) v4 -- 0x7f234296a100 con 0
 -1586> 2017-11-05 21:59:16.296837 7f232c2dc700 10 _calc_signature seq 19 front_crc_ = 3209079344 middle_crc = 0 data_crc = 0 sig = 179244109618777490
 -1585> 2017-11-05 21:59:16.296861 7f232c2dc700 20 Putting signature in client message(seq # 19): sig = 179244109618777490
 -1584> 2017-11-05 21:59:16.332743 7f232c2dc700 10 _calc_signature seq 248515247 front_crc_ = 822332401 middle_crc = 0 data_crc = 0 sig = 15675063173283232976
 -1583> 2017-11-05 21:59:16.332759 7f232c2dc700  5 -- 10.10.1.23:6789/0 >> 10.10.1.21:6789/0 conn(0x7f2342dd4800 :-1 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=1124301 cs=1 l=0). rx mon.0 seq 248515247 0x7f234296a100 paxos(commit lc 112907743 fc 0 pn 13700 opn 0) v4
 -1582> 2017-11-05 21:59:16.332820 7f23302e4700  1 -- 10.10.1.23:6789/0 <== mon.0 10.10.1.21:6789/0 248515247 ==== paxos(commit lc 112907743 fc 0 pn 13700 opn 0) v4 ==== 3368+0+0 (822332401 0 0) 0x7f234296a100 con 0x7f2342dd4800
 -1581> 2017-11-05 21:59:16.332852 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
 -1580> 2017-11-05 21:59:16.332861 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
 -1579> 2017-11-05 21:59:16.332864 7f23302e4700 20 is_capable service=mon command= read on cap allow *
 -1578> 2017-11-05 21:59:16.332867 7f23302e4700 20  allow so far , doing grant allow *
 -1577> 2017-11-05 21:59:16.332868 7f23302e4700 20  allow all
 -1576> 2017-11-05 21:59:16.332869 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
 -1575> 2017-11-05 21:59:16.332871 7f23302e4700 20  allow so far , doing grant allow *
 -1574> 2017-11-05 21:59:16.332872 7f23302e4700 20  allow all
 -1573> 2017-11-05 21:59:16.332876 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos updating c 112907080..112907742) handle_commit on 112907743
 -1572> 2017-11-05 21:59:16.332882 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos updating c 112907080..112907743) store_state [112907743..112907743]
 -1571> 2017-11-05 21:59:16.333041 7f232c2dc700 10 _calc_signature seq 248515248 front_crc_ = 3386463811 middle_crc = 0 data_crc = 0 sig = 14945621050292252831
 -1570> 2017-11-05 21:59:16.333052 7f232c2dc700  5 -- 10.10.1.23:6789/0 >> 10.10.1.21:6789/0 conn(0x7f2342dd4800 :-1 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=1124301 cs=1 l=0). rx mon.0 seq 248515248 0x7f2342969e00 paxos(lease lc 112907743 fc 112907080 pn 0 opn 0) v4
 -1569> 2017-11-05 21:59:16.374435 7f23302e4700 10 mon.ceph03@2(peon) e14 refresh_from_paxos
 -1568> 2017-11-05 21:59:16.374470 7f23302e4700 10 mon.ceph03@2(peon) e14 refresh_from_paxos no cluster_fingerprint
 -1567> 2017-11-05 21:59:16.374504 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(pgmap 56600067..56600783) refresh
 -1566> 2017-11-05 21:59:16.374545 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mdsmap 1..1) refresh
 -1565> 2017-11-05 21:59:16.374590 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(osdmap 503..1105) refresh
 -1564> 2017-11-05 21:59:16.374623 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(logm 55596915..55597635) refresh
 -1563> 2017-11-05 21:59:16.374629 7f23302e4700 10 mon.ceph03@2(peon).log v55597635 update_from_paxos
 -1562> 2017-11-05 21:59:16.374631 7f23302e4700 10 mon.ceph03@2(peon).log v55597635 update_from_paxos version 55597635 summary v 55597635
 -1561> 2017-11-05 21:59:16.374670 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(monmap 1..14) refresh
 -1560> 2017-11-05 21:59:16.374699 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(auth 26252..26331) refresh
 -1559> 2017-11-05 21:59:16.374705 7f23302e4700 10 mon.ceph03@2(peon).auth v26331 update_from_paxos
 -1558> 2017-11-05 21:59:16.374744 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgr 1..394) refresh
 -1557> 2017-11-05 21:59:16.374757 7f23302e4700  4 mgrc handle_mgr_map Got map version 394
 -1556> 2017-11-05 21:59:16.374760 7f23302e4700  4 mgrc handle_mgr_map Active mgr is now 10.10.1.21:6806/22556
 -1555> 2017-11-05 21:59:16.374798 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgrstat 56609513..56609636) refresh
 -1554> 2017-11-05 21:59:16.374804 7f23302e4700 10 mon.ceph03@2(peon).mgrstat  56609636
 -1553> 2017-11-05 21:59:16.374847 7f23302e4700 10 mon.ceph03@2(peon).mgrstat update_from_paxos v56609636 service_map e1
 -1552> 2017-11-05 21:59:16.374852 7f23302e4700 10 mon.ceph03@2(peon).mgrstat check_subs
 -1551> 2017-11-05 21:59:16.374854 7f23302e4700 20 mon.ceph03@2(peon).mgrstat update_logger
 -1550> 2017-11-05 21:59:16.374884 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(health 1..23) refresh
 -1549> 2017-11-05 21:59:16.374892 7f23302e4700 10 mon.ceph03@2(peon).health update_from_paxos
 -1548> 2017-11-05 21:59:16.374913 7f23302e4700 20 mon.ceph03@2(peon).health dump:{
    "quorum_health": {},
    "leader_health": {
        "MON_DOWN": {
            "severity": "HEALTH_WARN",
            "summary": {
                "message": "1/3 mons down, quorum ceph01mon,ceph02mon"
            },
            "detail": [
                {
                    "message": "mon.ceph03 (rank 2) addr 10.10.1.23:6789/0 is down (out of quorum)"
                }
            ]
        }
    }
}

 -1547> 2017-11-05 21:59:16.374935 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(pgmap 56600067..56600783) post_refresh
 -1546> 2017-11-05 21:59:16.374937 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mdsmap 1..1) post_refresh
 -1545> 2017-11-05 21:59:16.374939 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(osdmap 503..1105) post_refresh
 -1544> 2017-11-05 21:59:16.374940 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(logm 55596915..55597635) post_refresh
 -1543> 2017-11-05 21:59:16.374942 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(monmap 1..14) post_refresh
 -1542> 2017-11-05 21:59:16.374943 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(auth 26252..26331) post_refresh
 -1541> 2017-11-05 21:59:16.374945 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgr 1..394) post_refresh
 -1540> 2017-11-05 21:59:16.374946 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgrstat 56609513..56609636) post_refresh
 -1539> 2017-11-05 21:59:16.374949 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(health 1..23) post_refresh
 -1538> 2017-11-05 21:59:16.375012 7f23302e4700  1 -- 10.10.1.23:6789/0 <== mon.0 10.10.1.21:6789/0 248515248 ==== paxos(lease lc 112907743 fc 112907080 pn 0 opn 0) v4 ==== 84+0+0 (3386463811 0 0) 0x7f2342969e00 con 0x7f2342dd4800
 -1537> 2017-11-05 21:59:16.375031 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
 -1536> 2017-11-05 21:59:16.375041 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
 -1535> 2017-11-05 21:59:16.375044 7f23302e4700 20 is_capable service=mon command= read on cap allow *
 -1534> 2017-11-05 21:59:16.375047 7f23302e4700 20  allow so far , doing grant allow *
 -1533> 2017-11-05 21:59:16.375062 7f23302e4700 20  allow all
 -1532> 2017-11-05 21:59:16.375064 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
 -1531> 2017-11-05 21:59:16.375066 7f23302e4700 20  allow so far , doing grant allow *
 -1530> 2017-11-05 21:59:16.375067 7f23302e4700 20  allow all
 -1529> 2017-11-05 21:59:16.375072 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907743) handle_lease on 112907743 now 2017-11-05 21:59:21.331105
 -1528> 2017-11-05 21:59:16.375085 7f23302e4700  1 -- 10.10.1.23:6789/0 --> 10.10.1.21:6789/0 -- paxos(lease_ack lc 112907743 fc 112907080 pn 0 opn 0) v4 -- 0x7f234296a100 con 0
 -1527> 2017-11-05 21:59:16.375106 7f23302e4700 20 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907743) reset_lease_timeout - setting timeout event
 -1526> 2017-11-05 21:59:16.375116 7f23302e4700  5 mon.ceph03@2(peon).monmap v14 apply_mon_features wait for service to be writeable
 -1525> 2017-11-05 21:59:16.375120 7f23302e4700  5 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907743) is_readable = 1 - now=2017-11-05 21:59:16.375120 lease_expire=2017-11-05 21:59:21.331105 has v0 lc 112907743
 -1524> 2017-11-05 21:59:16.375160 7f232c2dc700 10 _calc_signature seq 20 front_crc_ = 3298296992 middle_crc = 0 data_crc = 0 sig = 17385314259900435840
 -1523> 2017-11-05 21:59:16.375177 7f232c2dc700 20 Putting signature in client message(seq # 20): sig = 17385314259900435840
 -1522> 2017-11-05 21:59:16.797649 7f232c2dc700 10 _calc_signature seq 248515249 front_crc_ = 1719413385 middle_crc = 0 data_crc = 0 sig = 6927146818612255408
 -1521> 2017-11-05 21:59:16.797669 7f232c2dc700  5 -- 10.10.1.23:6789/0 >> 10.10.1.21:6789/0 conn(0x7f2342dd4800 :-1 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=1124301 cs=1 l=0). rx mon.0 seq 248515249 0x7f234296a100 paxos(begin lc 112907743 fc 0 pn 13700 opn 0) v4
 -1520> 2017-11-05 21:59:16.797713 7f23302e4700  1 -- 10.10.1.23:6789/0 <== mon.0 10.10.1.21:6789/0 248515249 ==== paxos(begin lc 112907743 fc 0 pn 13700 opn 0) v4 ==== 302+0+0 (1719413385 0 0) 0x7f234296a100 con 0x7f2342dd4800
 -1519> 2017-11-05 21:59:16.797744 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
 -1518> 2017-11-05 21:59:16.797754 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
 -1517> 2017-11-05 21:59:16.797758 7f23302e4700 20 is_capable service=mon command= read on cap allow *
 -1516> 2017-11-05 21:59:16.797761 7f23302e4700 20  allow so far , doing grant allow *
 -1515> 2017-11-05 21:59:16.797763 7f23302e4700 20  allow all
 -1514> 2017-11-05 21:59:16.797765 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
 -1513> 2017-11-05 21:59:16.797766 7f23302e4700 20  allow so far , doing grant allow *
 -1512> 2017-11-05 21:59:16.797767 7f23302e4700 20  allow all
 -1511> 2017-11-05 21:59:16.797773 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907743) handle_begin paxos(begin lc 112907743 fc 0 pn 13700 opn 0) v4
 -1510> 2017-11-05 21:59:16.797778 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos updating c 112907080..112907743) accepting value for 112907744 pn 13700
 -1509> 2017-11-05 21:59:16.839583 7f23302e4700  1 -- 10.10.1.23:6789/0 --> 10.10.1.21:6789/0 -- paxos(accept lc 112907743 fc 0 pn 13700 opn 0) v4 -- 0x7f2342969e00 con 0
 -1508> 2017-11-05 21:59:16.839691 7f232c2dc700 10 _calc_signature seq 21 front_crc_ = 219823083 middle_crc = 0 data_crc = 0 sig = 16538990159248630615
 -1507> 2017-11-05 21:59:16.839710 7f232c2dc700 20 Putting signature in client message(seq # 21): sig = 16538990159248630615
 -1506> 2017-11-05 21:59:16.875406 7f232c2dc700 10 _calc_signature seq 248515250 front_crc_ = 3850737170 middle_crc = 0 data_crc = 0 sig = 17769774838374907603
 -1505> 2017-11-05 21:59:16.875423 7f232c2dc700  5 -- 10.10.1.23:6789/0 >> 10.10.1.21:6789/0 conn(0x7f2342dd4800 :-1 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=1124301 cs=1 l=0). rx mon.0 seq 248515250 0x7f2342969e00 paxos(commit lc 112907744 fc 0 pn 13700 opn 0) v4
 -1504> 2017-11-05 21:59:16.875464 7f23302e4700  1 -- 10.10.1.23:6789/0 <== mon.0 10.10.1.21:6789/0 248515250 ==== paxos(commit lc 112907744 fc 0 pn 13700 opn 0) v4 ==== 302+0+0 (3850737170 0 0) 0x7f2342969e00 con 0x7f2342dd4800
 -1503> 2017-11-05 21:59:16.875492 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
 -1502> 2017-11-05 21:59:16.875501 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
 -1501> 2017-11-05 21:59:16.875504 7f23302e4700 20 is_capable service=mon command= read on cap allow *
 -1500> 2017-11-05 21:59:16.875506 7f23302e4700 20  allow so far , doing grant allow *
 -1499> 2017-11-05 21:59:16.875508 7f23302e4700 20  allow all
 -1498> 2017-11-05 21:59:16.875509 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
 -1497> 2017-11-05 21:59:16.875513 7f23302e4700 20  allow so far , doing grant allow *
 -1496> 2017-11-05 21:59:16.875514 7f23302e4700 20  allow all
 -1495> 2017-11-05 21:59:16.875518 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos updating c 112907080..112907743) handle_commit on 112907744
 -1494> 2017-11-05 21:59:16.875523 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos updating c 112907080..112907744) store_state [112907744..112907744]
 -1493> 2017-11-05 21:59:16.875701 7f232c2dc700 10 _calc_signature seq 248515251 front_crc_ = 2110551307 middle_crc = 0 data_crc = 0 sig = 6423051929449659683
 -1492> 2017-11-05 21:59:16.875712 7f232c2dc700  5 -- 10.10.1.23:6789/0 >> 10.10.1.21:6789/0 conn(0x7f2342dd4800 :-1 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=1124301 cs=1 l=0). rx mon.0 seq 248515251 0x7f234296b000 paxos(lease lc 112907744 fc 112907080 pn 0 opn 0) v4
 -1491> 2017-11-05 21:59:17.005793 7f23302e4700 10 mon.ceph03@2(peon) e14 refresh_from_paxos
 -1490> 2017-11-05 21:59:17.005828 7f23302e4700 10 mon.ceph03@2(peon) e14 refresh_from_paxos no cluster_fingerprint
 -1489> 2017-11-05 21:59:17.005862 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(pgmap 56600067..56600783) refresh
 -1488> 2017-11-05 21:59:17.005906 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mdsmap 1..1) refresh
 -1487> 2017-11-05 21:59:17.005952 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(osdmap 503..1105) refresh
 -1486> 2017-11-05 21:59:17.005984 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(logm 55596915..55597635) refresh
 -1485> 2017-11-05 21:59:17.005990 7f23302e4700 10 mon.ceph03@2(peon).log v55597635 update_from_paxos
 -1484> 2017-11-05 21:59:17.005993 7f23302e4700 10 mon.ceph03@2(peon).log v55597635 update_from_paxos version 55597635 summary v 55597635
 -1483> 2017-11-05 21:59:17.006031 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(monmap 1..14) refresh
 -1482> 2017-11-05 21:59:17.006060 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(auth 26252..26331) refresh
 -1481> 2017-11-05 21:59:17.006066 7f23302e4700 10 mon.ceph03@2(peon).auth v26331 update_from_paxos
 -1480> 2017-11-05 21:59:17.006105 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgr 1..394) refresh
 -1479> 2017-11-05 21:59:17.006118 7f23302e4700  4 mgrc handle_mgr_map Got map version 394
 -1478> 2017-11-05 21:59:17.006121 7f23302e4700  4 mgrc handle_mgr_map Active mgr is now 10.10.1.21:6806/22556
 -1477> 2017-11-05 21:59:17.006160 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgrstat 56609513..56609636) refresh
 -1476> 2017-11-05 21:59:17.006168 7f23302e4700 10 mon.ceph03@2(peon).mgrstat  56609636
 -1475> 2017-11-05 21:59:17.006210 7f23302e4700 10 mon.ceph03@2(peon).mgrstat update_from_paxos v56609636 service_map e1
 -1474> 2017-11-05 21:59:17.006216 7f23302e4700 10 mon.ceph03@2(peon).mgrstat check_subs
 -1473> 2017-11-05 21:59:17.006218 7f23302e4700 20 mon.ceph03@2(peon).mgrstat update_logger
 -1472> 2017-11-05 21:59:17.006248 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(health 1..24) refresh
 -1471> 2017-11-05 21:59:17.006254 7f23302e4700 10 mon.ceph03@2(peon).health update_from_paxos
 -1470> 2017-11-05 21:59:17.006271 7f23302e4700 20 mon.ceph03@2(peon).health dump:{
    "quorum_health": {},
    "leader_health": {}
}

 -1469> 2017-11-05 21:59:17.006285 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(pgmap 56600067..56600783) post_refresh
 -1468> 2017-11-05 21:59:17.006288 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mdsmap 1..1) post_refresh
 -1467> 2017-11-05 21:59:17.006289 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(osdmap 503..1105) post_refresh
 -1466> 2017-11-05 21:59:17.006291 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(logm 55596915..55597635) post_refresh
 -1465> 2017-11-05 21:59:17.006292 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(monmap 1..14) post_refresh
 -1464> 2017-11-05 21:59:17.006294 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(auth 26252..26331) post_refresh
 -1463> 2017-11-05 21:59:17.006295 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgr 1..394) post_refresh
 -1462> 2017-11-05 21:59:17.006297 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgrstat 56609513..56609636) post_refresh
 -1461> 2017-11-05 21:59:17.006299 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(health 1..24) post_refresh
 -1460> 2017-11-05 21:59:17.006371 7f23302e4700  1 -- 10.10.1.23:6789/0 <== mon.0 10.10.1.21:6789/0 248515251 ==== paxos(lease lc 112907744 fc 112907080 pn 0 opn 0) v4 ==== 84+0+0 (2110551307 0 0) 0x7f234296b000 con 0x7f2342dd4800
 -1459> 2017-11-05 21:59:17.006390 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
 -1458> 2017-11-05 21:59:17.006398 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
 -1457> 2017-11-05 21:59:17.006402 7f23302e4700 20 is_capable service=mon command= read on cap allow *
 -1456> 2017-11-05 21:59:17.006405 7f23302e4700 20  allow so far , doing grant allow *
 -1455> 2017-11-05 21:59:17.006406 7f23302e4700 20  allow all
 -1454> 2017-11-05 21:59:17.006408 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
 -1453> 2017-11-05 21:59:17.006409 7f23302e4700 20  allow so far , doing grant allow *
 -1452> 2017-11-05 21:59:17.006410 7f23302e4700 20  allow all
 -1451> 2017-11-05 21:59:17.006414 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907744) handle_lease on 112907744 now 2017-11-05 21:59:21.873858
 -1450> 2017-11-05 21:59:17.006426 7f23302e4700  1 -- 10.10.1.23:6789/0 --> 10.10.1.21:6789/0 -- paxos(lease_ack lc 112907744 fc 112907080 pn 0 opn 0) v4 -- 0x7f2342969e00 con 0
 -1449> 2017-11-05 21:59:17.006444 7f23302e4700 20 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907744) reset_lease_timeout - setting timeout event
 -1448> 2017-11-05 21:59:17.006454 7f23302e4700  5 mon.ceph03@2(peon).monmap v14 apply_mon_features wait for service to be writeable
 -1447> 2017-11-05 21:59:17.006457 7f23302e4700  5 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907744) is_readable = 1 - now=2017-11-05 21:59:17.006458 lease_expire=2017-11-05 21:59:21.873858 has v0 lc 112907744
 -1446> 2017-11-05 21:59:17.006522 7f232c2dc700 10 _calc_signature seq 22 front_crc_ = 978136004 middle_crc = 0 data_crc = 0 sig = 7430388262177812061
 -1445> 2017-11-05 21:59:17.006540 7f232c2dc700 20 Putting signature in client message(seq # 22): sig = 7430388262177812061
 -1444> 2017-11-05 21:59:17.374028 7f232c2dc700 10 _calc_signature seq 248515252 front_crc_ = 2233000939 middle_crc = 0 data_crc = 0 sig = 5913206972756699187
 -1443> 2017-11-05 21:59:17.374050 7f232c2dc700  5 -- 10.10.1.23:6789/0 >> 10.10.1.21:6789/0 conn(0x7f2342dd4800 :-1 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=1124301 cs=1 l=0). rx mon.0 seq 248515252 0x7f2342969e00 paxos(begin lc 112907744 fc 0 pn 13700 opn 0) v4
 -1442> 2017-11-05 21:59:17.374115 7f23302e4700  1 -- 10.10.1.23:6789/0 <== mon.0 10.10.1.21:6789/0 248515252 ==== paxos(begin lc 112907744 fc 0 pn 13700 opn 0) v4 ==== 9833+0+0 (2233000939 0 0) 0x7f2342969e00 con 0x7f2342dd4800
 -1441> 2017-11-05 21:59:17.374152 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
 -1440> 2017-11-05 21:59:17.374163 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
 -1439> 2017-11-05 21:59:17.374167 7f23302e4700 20 is_capable service=mon command= read on cap allow *
 -1438> 2017-11-05 21:59:17.374170 7f23302e4700 20  allow so far , doing grant allow *
 -1437> 2017-11-05 21:59:17.374172 7f23302e4700 20  allow all
 -1436> 2017-11-05 21:59:17.374174 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
 -1435> 2017-11-05 21:59:17.374175 7f23302e4700 20  allow so far , doing grant allow *
 -1434> 2017-11-05 21:59:17.374176 7f23302e4700 20  allow all
 -1433> 2017-11-05 21:59:17.374182 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907744) handle_begin paxos(begin lc 112907744 fc 0 pn 13700 opn 0) v4
 -1432> 2017-11-05 21:59:17.374189 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos updating c 112907080..112907744) accepting value for 112907745 pn 13700
 -1431> 2017-11-05 21:59:17.415481 7f23302e4700  1 -- 10.10.1.23:6789/0 --> 10.10.1.21:6789/0 -- paxos(accept lc 112907744 fc 0 pn 13700 opn 0) v4 -- 0x7f234296b000 con 0
 -1430> 2017-11-05 21:59:17.415602 7f232c2dc700 10 _calc_signature seq 23 front_crc_ = 822198047 middle_crc = 0 data_crc = 0 sig = 11283865087405808684
 -1429> 2017-11-05 21:59:17.415619 7f232c2dc700 20 Putting signature in client message(seq # 23): sig = 11283865087405808684
 -1428> 2017-11-05 21:59:17.452463 7f232c2dc700 10 _calc_signature seq 248515253 front_crc_ = 4262535181 middle_crc = 0 data_crc = 0 sig = 9454139455262504054
 -1427> 2017-11-05 21:59:17.452480 7f232c2dc700  5 -- 10.10.1.23:6789/0 >> 10.10.1.21:6789/0 conn(0x7f2342dd4800 :-1 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=1124301 cs=1 l=0). rx mon.0 seq 248515253 0x7f234296b000 paxos(commit lc 112907745 fc 0 pn 13700 opn 0) v4
 -1426> 2017-11-05 21:59:17.452517 7f232c2dc700 10 _calc_signature seq 248515254 front_crc_ = 3114647245 middle_crc = 0 data_crc = 0 sig = 1786097533032395933
 -1425> 2017-11-05 21:59:17.452521 7f232c2dc700  5 -- 10.10.1.23:6789/0 >> 10.10.1.21:6789/0 conn(0x7f2342dd4800 :-1 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=1124301 cs=1 l=0). rx mon.0 seq 248515254 0x7f234296ad00 paxos(lease lc 112907745 fc 112907080 pn 0 opn 0) v4
 -1424> 2017-11-05 21:59:17.452540 7f23302e4700  1 -- 10.10.1.23:6789/0 <== mon.0 10.10.1.21:6789/0 248515253 ==== paxos(commit lc 112907745 fc 0 pn 13700 opn 0) v4 ==== 9833+0+0 (4262535181 0 0) 0x7f234296b000 con 0x7f2342dd4800
 -1423> 2017-11-05 21:59:17.452572 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
 -1422> 2017-11-05 21:59:17.452582 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
 -1421> 2017-11-05 21:59:17.452585 7f23302e4700 20 is_capable service=mon command= read on cap allow *
 -1420> 2017-11-05 21:59:17.452588 7f23302e4700 20  allow so far , doing grant allow *
 -1419> 2017-11-05 21:59:17.452589 7f23302e4700 20  allow all
 -1418> 2017-11-05 21:59:17.452591 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
 -1417> 2017-11-05 21:59:17.452593 7f23302e4700 20  allow so far , doing grant allow *
 -1416> 2017-11-05 21:59:17.452594 7f23302e4700 20  allow all
 -1415> 2017-11-05 21:59:17.452599 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos updating c 112907080..112907744) handle_commit on 112907745
 -1414> 2017-11-05 21:59:17.452604 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos updating c 112907080..112907745) store_state [112907745..112907745]
 -1413> 2017-11-05 21:59:17.493220 7f23302e4700 10 mon.ceph03@2(peon) e14 refresh_from_paxos
 -1412> 2017-11-05 21:59:17.493254 7f23302e4700 10 mon.ceph03@2(peon) e14 refresh_from_paxos no cluster_fingerprint
 -1411> 2017-11-05 21:59:17.493288 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(pgmap 56600067..56600783) refresh
 -1410> 2017-11-05 21:59:17.493329 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mdsmap 1..1) refresh
 -1409> 2017-11-05 21:59:17.493374 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(osdmap 503..1105) refresh
 -1408> 2017-11-05 21:59:17.493407 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(logm 55596915..55597636) refresh
 -1407> 2017-11-05 21:59:17.493414 7f23302e4700 10 mon.ceph03@2(peon).log v55597636 update_from_paxos
 -1406> 2017-11-05 21:59:17.493417 7f23302e4700 10 mon.ceph03@2(peon).log v55597636 update_from_paxos version 55597636 summary v 55597635
 -1405> 2017-11-05 21:59:17.493425 7f23302e4700 10 mon.ceph03@2(peon).log v55597636 update_from_paxos latest full 55597635
 -1404> 2017-11-05 21:59:17.493441 7f23302e4700  7 mon.ceph03@2(peon).log v55597636 update_from_paxos applying incremental log 55597636 2017-11-05 21:59:16.764738 mon.ceph01mon mon.0 10.10.1.21:6789/0 864 : cluster [INF] Health check cleared: MON_DOWN (was: 1/3 mons down, quorum ceph01mon,ceph02mon)
 -1403> 2017-11-05 21:59:17.493463 7f23302e4700 20 mon.ceph03@2(peon).log v55597636 update_from_paxos logging for channel 'cluster' to file '/var/log/ceph/ceph.log'
 -1402> 2017-11-05 21:59:17.493490 7f23302e4700 15 mon.ceph03@2(peon).log v55597636 update_from_paxos logging for 1 channels
 -1401> 2017-11-05 21:59:17.493496 7f23302e4700 15 mon.ceph03@2(peon).log v55597636 update_from_paxos channel 'cluster' logging 165 bytes
 -1400> 2017-11-05 21:59:17.493551 7f23302e4700 10 mon.ceph03@2(peon).log v55597636 check_subs
 -1399> 2017-11-05 21:59:17.493604 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(monmap 1..14) refresh
 -1398> 2017-11-05 21:59:17.493635 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(auth 26252..26331) refresh
 -1397> 2017-11-05 21:59:17.493641 7f23302e4700 10 mon.ceph03@2(peon).auth v26331 update_from_paxos
 -1396> 2017-11-05 21:59:17.493681 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgr 1..394) refresh
 -1395> 2017-11-05 21:59:17.493693 7f23302e4700  4 mgrc handle_mgr_map Got map version 394
 -1394> 2017-11-05 21:59:17.493696 7f23302e4700  4 mgrc handle_mgr_map Active mgr is now 10.10.1.21:6806/22556
 -1393> 2017-11-05 21:59:17.493734 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgrstat 56609513..56609636) refresh
 -1392> 2017-11-05 21:59:17.493740 7f23302e4700 10 mon.ceph03@2(peon).mgrstat  56609636
 -1391> 2017-11-05 21:59:17.493788 7f23302e4700 10 mon.ceph03@2(peon).mgrstat update_from_paxos v56609636 service_map e1
 -1390> 2017-11-05 21:59:17.493793 7f23302e4700 10 mon.ceph03@2(peon).mgrstat check_subs
 -1389> 2017-11-05 21:59:17.493795 7f23302e4700 20 mon.ceph03@2(peon).mgrstat update_logger
 -1388> 2017-11-05 21:59:17.493826 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(health 1..24) refresh
 -1387> 2017-11-05 21:59:17.493832 7f23302e4700 10 mon.ceph03@2(peon).health update_from_paxos
 -1386> 2017-11-05 21:59:17.493849 7f23302e4700 20 mon.ceph03@2(peon).health dump:{
    "quorum_health": {},
    "leader_health": {}
}

 -1385> 2017-11-05 21:59:17.493863 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(pgmap 56600067..56600783) post_refresh
 -1384> 2017-11-05 21:59:17.493866 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mdsmap 1..1) post_refresh
 -1383> 2017-11-05 21:59:17.493868 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(osdmap 503..1105) post_refresh
 -1382> 2017-11-05 21:59:17.493870 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(logm 55596915..55597636) post_refresh
 -1381> 2017-11-05 21:59:17.493872 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(monmap 1..14) post_refresh
 -1380> 2017-11-05 21:59:17.493874 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(auth 26252..26331) post_refresh
 -1379> 2017-11-05 21:59:17.493875 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgr 1..394) post_refresh
 -1378> 2017-11-05 21:59:17.493879 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgrstat 56609513..56609636) post_refresh
 -1377> 2017-11-05 21:59:17.493881 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(health 1..24) post_refresh
 -1376> 2017-11-05 21:59:17.493947 7f23302e4700  1 -- 10.10.1.23:6789/0 <== mon.0 10.10.1.21:6789/0 248515254 ==== paxos(lease lc 112907745 fc 112907080 pn 0 opn 0) v4 ==== 84+0+0 (3114647245 0 0) 0x7f234296ad00 con 0x7f2342dd4800
 -1375> 2017-11-05 21:59:17.493965 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
 -1374> 2017-11-05 21:59:17.493973 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
 -1373> 2017-11-05 21:59:17.493976 7f23302e4700 20 is_capable service=mon command= read on cap allow *
 -1372> 2017-11-05 21:59:17.493978 7f23302e4700 20  allow so far , doing grant allow *
 -1371> 2017-11-05 21:59:17.493980 7f23302e4700 20  allow all
 -1370> 2017-11-05 21:59:17.493981 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
 -1369> 2017-11-05 21:59:17.493983 7f23302e4700 20  allow so far , doing grant allow *
 -1368> 2017-11-05 21:59:17.493984 7f23302e4700 20  allow all
 -1367> 2017-11-05 21:59:17.493989 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907745) handle_lease on 112907745 now 2017-11-05 21:59:22.450233
 -1366> 2017-11-05 21:59:17.493999 7f23302e4700  1 -- 10.10.1.23:6789/0 --> 10.10.1.21:6789/0 -- paxos(lease_ack lc 112907745 fc 112907080 pn 0 opn 0) v4 -- 0x7f234296b000 con 0
 -1365> 2017-11-05 21:59:17.494014 7f23302e4700 20 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907745) reset_lease_timeout - setting timeout event
 -1364> 2017-11-05 21:59:17.494021 7f23302e4700  5 mon.ceph03@2(peon).monmap v14 apply_mon_features wait for service to be writeable
 -1363> 2017-11-05 21:59:17.494028 7f23302e4700  5 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907745) is_readable = 1 - now=2017-11-05 21:59:17.494029 lease_expire=2017-11-05 21:59:22.450233 has v0 lc 112907745
 -1362> 2017-11-05 21:59:17.494086 7f232c2dc700 10 _calc_signature seq 24 front_crc_ = 2877252748 middle_crc = 0 data_crc = 0 sig = 15957257717912452231
 -1361> 2017-11-05 21:59:17.494104 7f232c2dc700 20 Putting signature in client message(seq # 24): sig = 15957257717912452231
 -1360> 2017-11-05 21:59:17.691997 7f2332ae9700 11 mon.ceph03@2(peon) e14 tick
 -1359> 2017-11-05 21:59:17.692020 7f2332ae9700 10 mon.ceph03@2(peon).mds e1
 -1358> 2017-11-05 21:59:17.692026 7f2332ae9700 10 mon.ceph03@2(peon).osd e1105 e1105: 9 total, 9 up, 9 in
 -1357> 2017-11-05 21:59:17.692030 7f2332ae9700 10 mon.ceph03@2(peon).log v55597636 log
 -1356> 2017-11-05 21:59:17.692033 7f2332ae9700 10 mon.ceph03@2(peon).auth v26331 auth
 -1355> 2017-11-05 21:59:17.692036 7f2332ae9700 10 mon.ceph03@2(peon).health tick
 -1354> 2017-11-05 21:59:17.692038 7f2332ae9700 20 mon.ceph03@2(peon).health check_member_health
 -1353> 2017-11-05 21:59:17.692150 7f2332ae9700 10 mon.ceph03@2(peon).health check_member_health avail 92% total 277 GB, used 5676 MB, avail 257 GB
 -1352> 2017-11-05 21:59:17.692164 7f2332ae9700 20 mon.ceph03@2(peon) e14 sync_trim_providers
 -1351> 2017-11-05 21:59:17.897994 7f232f2e2700  1 -- 10.10.1.23:0/15195 --> 10.10.1.21:6806/22556 -- mgrreport(unknown.ceph03 +23-0 packed 1734) v4 -- 0x7f2342beb8c0 con 0
 -1350> 2017-11-05 21:59:17.898127 7f232d2de700 10 _calc_signature seq 3 front_crc_ = 1126695011 middle_crc = 0 data_crc = 0 sig = 13447063656664490716
 -1349> 2017-11-05 21:59:17.898145 7f232d2de700 20 Putting signature in client message(seq # 3): sig = 13447063656664490716
 -1348> 2017-11-05 21:59:18.481544 7f232c2dc700 10 _calc_signature seq 248515255 front_crc_ = 2335337029 middle_crc = 0 data_crc = 0 sig = 2844489569343748739
 -1347> 2017-11-05 21:59:18.481565 7f232c2dc700  5 -- 10.10.1.23:6789/0 >> 10.10.1.21:6789/0 conn(0x7f2342dd4800 :-1 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=1124301 cs=1 l=0). rx mon.0 seq 248515255 0x7f234296b000 paxos(begin lc 112907745 fc 0 pn 13700 opn 0) v4
 -1346> 2017-11-05 21:59:18.481641 7f23302e4700  1 -- 10.10.1.23:6789/0 <== mon.0 10.10.1.21:6789/0 248515255 ==== paxos(begin lc 112907745 fc 0 pn 13700 opn 0) v4 ==== 3368+0+0 (2335337029 0 0) 0x7f234296b000 con 0x7f2342dd4800
 -1345> 2017-11-05 21:59:18.481691 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
 -1344> 2017-11-05 21:59:18.481702 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
 -1343> 2017-11-05 21:59:18.481706 7f23302e4700 20 is_capable service=mon command= read on cap allow *
 -1342> 2017-11-05 21:59:18.481709 7f23302e4700 20  allow so far , doing grant allow *
 -1341> 2017-11-05 21:59:18.481710 7f23302e4700 20  allow all
 -1340> 2017-11-05 21:59:18.481713 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
 -1339> 2017-11-05 21:59:18.481714 7f23302e4700 20  allow so far , doing grant allow *
 -1338> 2017-11-05 21:59:18.481715 7f23302e4700 20  allow all
 -1337> 2017-11-05 21:59:18.481723 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907745) handle_begin paxos(begin lc 112907745 fc 0 pn 13700 opn 0) v4
 -1336> 2017-11-05 21:59:18.481728 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos updating c 112907080..112907745) accepting value for 112907746 pn 13700
 -1335> 2017-11-05 21:59:18.522966 7f23302e4700  1 -- 10.10.1.23:6789/0 --> 10.10.1.21:6789/0 -- paxos(accept lc 112907745 fc 0 pn 13700 opn 0) v4 -- 0x7f234296ad00 con 0
 -1334> 2017-11-05 21:59:18.523045 7f232c2dc700 10 _calc_signature seq 25 front_crc_ = 4275636177 middle_crc = 0 data_crc = 0 sig = 14765727533992502905
 -1333> 2017-11-05 21:59:18.523060 7f232c2dc700 20 Putting signature in client message(seq # 25): sig = 14765727533992502905
 -1332> 2017-11-05 21:59:18.559375 7f232c2dc700 10 _calc_signature seq 248515256 front_crc_ = 2988239189 middle_crc = 0 data_crc = 0 sig = 4256597801313242484
 -1331> 2017-11-05 21:59:18.559393 7f232c2dc700  5 -- 10.10.1.23:6789/0 >> 10.10.1.21:6789/0 conn(0x7f2342dd4800 :-1 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=1124301 cs=1 l=0). rx mon.0 seq 248515256 0x7f234296ad00 paxos(commit lc 112907746 fc 0 pn 13700 opn 0) v4
 -1330> 2017-11-05 21:59:18.559454 7f23302e4700  1 -- 10.10.1.23:6789/0 <== mon.0 10.10.1.21:6789/0 248515256 ==== paxos(commit lc 112907746 fc 0 pn 13700 opn 0) v4 ==== 3368+0+0 (2988239189 0 0) 0x7f234296ad00 con 0x7f2342dd4800
 -1329> 2017-11-05 21:59:18.559489 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
 -1328> 2017-11-05 21:59:18.559498 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
 -1327> 2017-11-05 21:59:18.559502 7f23302e4700 20 is_capable service=mon command= read on cap allow *
 -1326> 2017-11-05 21:59:18.559505 7f23302e4700 20  allow so far , doing grant allow *
 -1325> 2017-11-05 21:59:18.559508 7f23302e4700 20  allow all
 -1324> 2017-11-05 21:59:18.559510 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
 -1323> 2017-11-05 21:59:18.559511 7f23302e4700 20  allow so far , doing grant allow *
 -1322> 2017-11-05 21:59:18.559513 7f23302e4700 20  allow all
 -1321> 2017-11-05 21:59:18.559518 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos updating c 112907080..112907745) handle_commit on 112907746
 -1320> 2017-11-05 21:59:18.559523 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos updating c 112907080..112907746) store_state [112907746..112907746]
 -1319> 2017-11-05 21:59:18.559666 7f232c2dc700 10 _calc_signature seq 248515257 front_crc_ = 1949826167 middle_crc = 0 data_crc = 0 sig = 1217847151241602786
 -1318> 2017-11-05 21:59:18.559685 7f232c2dc700  5 -- 10.10.1.23:6789/0 >> 10.10.1.21:6789/0 conn(0x7f2342dd4800 :-1 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=1124301 cs=1 l=0). rx mon.0 seq 248515257 0x7f234296aa00 paxos(lease lc 112907746 fc 112907080 pn 0 opn 0) v4
 -1317> 2017-11-05 21:59:18.667111 7f23302e4700 10 mon.ceph03@2(peon) e14 refresh_from_paxos
 -1316> 2017-11-05 21:59:18.667147 7f23302e4700 10 mon.ceph03@2(peon) e14 refresh_from_paxos no cluster_fingerprint
 -1315> 2017-11-05 21:59:18.667185 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(pgmap 56600067..56600783) refresh
 -1314> 2017-11-05 21:59:18.667231 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mdsmap 1..1) refresh
 -1313> 2017-11-05 21:59:18.667280 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(osdmap 503..1105) refresh
 -1312> 2017-11-05 21:59:18.667315 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(logm 55596915..55597636) refresh
 -1311> 2017-11-05 21:59:18.667321 7f23302e4700 10 mon.ceph03@2(peon).log v55597636 update_from_paxos
 -1310> 2017-11-05 21:59:18.667326 7f23302e4700 10 mon.ceph03@2(peon).log v55597636 update_from_paxos version 55597636 summary v 55597636
 -1309> 2017-11-05 21:59:18.667367 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(monmap 1..14) refresh
 -1308> 2017-11-05 21:59:18.667399 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(auth 26252..26331) refresh
 -1307> 2017-11-05 21:59:18.667405 7f23302e4700 10 mon.ceph03@2(peon).auth v26331 update_from_paxos
 -1306> 2017-11-05 21:59:18.667448 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgr 1..394) refresh
 -1305> 2017-11-05 21:59:18.667465 7f23302e4700  4 mgrc handle_mgr_map Got map version 394
 -1304> 2017-11-05 21:59:18.667473 7f23302e4700  4 mgrc handle_mgr_map Active mgr is now 10.10.1.21:6806/22556
 -1303> 2017-11-05 21:59:18.667515 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgrstat 56609513..56609637) refresh
 -1302> 2017-11-05 21:59:18.667523 7f23302e4700 10 mon.ceph03@2(peon).mgrstat  56609637
 -1301> 2017-11-05 21:59:18.667578 7f23302e4700 10 mon.ceph03@2(peon).mgrstat update_from_paxos v56609637 service_map e1
 -1300> 2017-11-05 21:59:18.667584 7f23302e4700 10 mon.ceph03@2(peon).mgrstat check_subs
 -1299> 2017-11-05 21:59:18.667587 7f23302e4700 20 mon.ceph03@2(peon).mgrstat update_logger
 -1298> 2017-11-05 21:59:18.667619 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(health 1..24) refresh
 -1297> 2017-11-05 21:59:18.667626 7f23302e4700 10 mon.ceph03@2(peon).health update_from_paxos
 -1296> 2017-11-05 21:59:18.667642 7f23302e4700 20 mon.ceph03@2(peon).health dump:{
    "quorum_health": {},
    "leader_health": {}
}

 -1295> 2017-11-05 21:59:18.667658 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(pgmap 56600067..56600783) post_refresh
 -1294> 2017-11-05 21:59:18.667661 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mdsmap 1..1) post_refresh
 -1293> 2017-11-05 21:59:18.667663 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(osdmap 503..1105) post_refresh
 -1292> 2017-11-05 21:59:18.667665 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(logm 55596915..55597636) post_refresh
 -1291> 2017-11-05 21:59:18.667667 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(monmap 1..14) post_refresh
 -1290> 2017-11-05 21:59:18.667669 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(auth 26252..26331) post_refresh
 -1289> 2017-11-05 21:59:18.667671 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgr 1..394) post_refresh
 -1288> 2017-11-05 21:59:18.667674 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgrstat 56609513..56609637) post_refresh
 -1287> 2017-11-05 21:59:18.667676 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(health 1..24) post_refresh
 -1286> 2017-11-05 21:59:18.667746 7f23302e4700  1 -- 10.10.1.23:6789/0 <== mon.0 10.10.1.21:6789/0 248515257 ==== paxos(lease lc 112907746 fc 112907080 pn 0 opn 0) v4 ==== 84+0+0 (1949826167 0 0) 0x7f234296aa00 con 0x7f2342dd4800
 -1285> 2017-11-05 21:59:18.667765 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
 -1284> 2017-11-05 21:59:18.667773 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
 -1283> 2017-11-05 21:59:18.667776 7f23302e4700 20 is_capable service=mon command= read on cap allow *
 -1282> 2017-11-05 21:59:18.667779 7f23302e4700 20  allow so far , doing grant allow *
 -1281> 2017-11-05 21:59:18.667780 7f23302e4700 20  allow all
 -1280> 2017-11-05 21:59:18.667782 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
 -1279> 2017-11-05 21:59:18.667784 7f23302e4700 20  allow so far , doing grant allow *
 -1278> 2017-11-05 21:59:18.667785 7f23302e4700 20  allow all
 -1277> 2017-11-05 21:59:18.667789 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907746) handle_lease on 112907746 now 2017-11-05 21:59:23.557741
 -1276> 2017-11-05 21:59:18.667803 7f23302e4700  1 -- 10.10.1.23:6789/0 --> 10.10.1.21:6789/0 -- paxos(lease_ack lc 112907746 fc 112907080 pn 0 opn 0) v4 -- 0x7f234296ad00 con 0
 -1275> 2017-11-05 21:59:18.667821 7f23302e4700 20 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907746) reset_lease_timeout - setting timeout event
 -1274> 2017-11-05 21:59:18.667832 7f23302e4700  5 mon.ceph03@2(peon).monmap v14 apply_mon_features wait for service to be writeable
 -1273> 2017-11-05 21:59:18.667835 7f23302e4700  5 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907746) is_readable = 1 - now=2017-11-05 21:59:18.667836 lease_expire=2017-11-05 21:59:23.557741 has v0 lc 112907746
 -1272> 2017-11-05 21:59:18.667886 7f232c2dc700 10 _calc_signature seq 26 front_crc_ = 3644507818 middle_crc = 0 data_crc = 0 sig = 7016679905100796258
 -1271> 2017-11-05 21:59:18.667903 7f232c2dc700 20 Putting signature in client message(seq # 26): sig = 7016679905100796258
 -1270> 2017-11-05 21:59:20.253574 7f232c2dc700 10 _calc_signature seq 248515258 front_crc_ = 2845299607 middle_crc = 0 data_crc = 0 sig = 14165744638036507123
 -1269> 2017-11-05 21:59:20.253594 7f232c2dc700  5 -- 10.10.1.23:6789/0 >> 10.10.1.21:6789/0 conn(0x7f2342dd4800 :-1 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=1124301 cs=1 l=0). rx mon.0 seq 248515258 0x7f234296ad00 paxos(begin lc 112907746 fc 0 pn 13700 opn 0) v4
 -1268> 2017-11-05 21:59:20.253681 7f23302e4700  1 -- 10.10.1.23:6789/0 <== mon.0 10.10.1.21:6789/0 248515258 ==== paxos(begin lc 112907746 fc 0 pn 13700 opn 0) v4 ==== 3360+0+0 (2845299607 0 0) 0x7f234296ad00 con 0x7f2342dd4800
 -1267> 2017-11-05 21:59:20.253717 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
 -1266> 2017-11-05 21:59:20.253730 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
 -1265> 2017-11-05 21:59:20.253735 7f23302e4700 20 is_capable service=mon command= read on cap allow *
 -1264> 2017-11-05 21:59:20.253738 7f23302e4700 20  allow so far , doing grant allow *
 -1263> 2017-11-05 21:59:20.253740 7f23302e4700 20  allow all
 -1262> 2017-11-05 21:59:20.253742 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
 -1261> 2017-11-05 21:59:20.253744 7f23302e4700 20  allow so far , doing grant allow *
 -1260> 2017-11-05 21:59:20.253745 7f23302e4700 20  allow all
 -1259> 2017-11-05 21:59:20.253751 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907746) handle_begin paxos(begin lc 112907746 fc 0 pn 13700 opn 0) v4
 -1258> 2017-11-05 21:59:20.253756 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos updating c 112907080..112907746) accepting value for 112907747 pn 13700
 -1257> 2017-11-05 21:59:20.294496 7f23302e4700  1 -- 10.10.1.23:6789/0 --> 10.10.1.21:6789/0 -- paxos(accept lc 112907746 fc 0 pn 13700 opn 0) v4 -- 0x7f234296aa00 con 0
 -1256> 2017-11-05 21:59:20.294606 7f232c2dc700 10 _calc_signature seq 27 front_crc_ = 2216739829 middle_crc = 0 data_crc = 0 sig = 14551920355953459298
 -1255> 2017-11-05 21:59:20.294651 7f232c2dc700 20 Putting signature in client message(seq # 27): sig = 14551920355953459298
 -1254> 2017-11-05 21:59:20.331167 7f232c2dc700 10 _calc_signature seq 248515259 front_crc_ = 2227490693 middle_crc = 0 data_crc = 0 sig = 2808689771223304806
 -1253> 2017-11-05 21:59:20.331200 7f232c2dc700  5 -- 10.10.1.23:6789/0 >> 10.10.1.21:6789/0 conn(0x7f2342dd4800 :-1 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=1124301 cs=1 l=0). rx mon.0 seq 248515259 0x7f234296aa00 paxos(commit lc 112907747 fc 0 pn 13700 opn 0) v4
 -1252> 2017-11-05 21:59:20.331263 7f23302e4700  1 -- 10.10.1.23:6789/0 <== mon.0 10.10.1.21:6789/0 248515259 ==== paxos(commit lc 112907747 fc 0 pn 13700 opn 0) v4 ==== 3360+0+0 (2227490693 0 0) 0x7f234296aa00 con 0x7f2342dd4800
 -1251> 2017-11-05 21:59:20.331308 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
 -1250> 2017-11-05 21:59:20.331318 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
 -1249> 2017-11-05 21:59:20.331326 7f23302e4700 20 is_capable service=mon command= read on cap allow *
 -1248> 2017-11-05 21:59:20.331328 7f23302e4700 20  allow so far , doing grant allow *
 -1247> 2017-11-05 21:59:20.331329 7f23302e4700 20  allow all
 -1246> 2017-11-05 21:59:20.331331 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
 -1245> 2017-11-05 21:59:20.331332 7f23302e4700 20  allow so far , doing grant allow *
 -1244> 2017-11-05 21:59:20.331333 7f23302e4700 20  allow all
 -1243> 2017-11-05 21:59:20.331338 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos updating c 112907080..112907746) handle_commit on 112907747
 -1242> 2017-11-05 21:59:20.331343 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos updating c 112907080..112907747) store_state [112907747..112907747]
 -1241> 2017-11-05 21:59:20.331448 7f232c2dc700 10 _calc_signature seq 248515260 front_crc_ = 618269879 middle_crc = 0 data_crc = 0 sig = 18220854632036809290
 -1240> 2017-11-05 21:59:20.331460 7f232c2dc700  5 -- 10.10.1.23:6789/0 >> 10.10.1.21:6789/0 conn(0x7f2342dd4800 :-1 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=1124301 cs=1 l=0). rx mon.0 seq 248515260 0x7f234296a700 paxos(lease lc 112907747 fc 112907080 pn 0 opn 0) v4
 -1239> 2017-11-05 21:59:20.372209 7f23302e4700 10 mon.ceph03@2(peon) e14 refresh_from_paxos
 -1238> 2017-11-05 21:59:20.372243 7f23302e4700 10 mon.ceph03@2(peon) e14 refresh_from_paxos no cluster_fingerprint
 -1237> 2017-11-05 21:59:20.372281 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(pgmap 56600067..56600783) refresh
 -1236> 2017-11-05 21:59:20.372332 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mdsmap 1..1) refresh
 -1235> 2017-11-05 21:59:20.372380 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(osdmap 503..1105) refresh
 -1234> 2017-11-05 21:59:20.372427 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(logm 55596915..55597636) refresh
 -1233> 2017-11-05 21:59:20.372433 7f23302e4700 10 mon.ceph03@2(peon).log v55597636 update_from_paxos
 -1232> 2017-11-05 21:59:20.372435 7f23302e4700 10 mon.ceph03@2(peon).log v55597636 update_from_paxos version 55597636 summary v 55597636
 -1231> 2017-11-05 21:59:20.372474 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(monmap 1..14) refresh
 -1230> 2017-11-05 21:59:20.372504 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(auth 26252..26331) refresh
 -1229> 2017-11-05 21:59:20.372510 7f23302e4700 10 mon.ceph03@2(peon).auth v26331 update_from_paxos
 -1228> 2017-11-05 21:59:20.372550 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgr 1..394) refresh
 -1227> 2017-11-05 21:59:20.372567 7f23302e4700  4 mgrc handle_mgr_map Got map version 394
 -1226> 2017-11-05 21:59:20.372573 7f23302e4700  4 mgrc handle_mgr_map Active mgr is now 10.10.1.21:6806/22556
 -1225> 2017-11-05 21:59:20.372612 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgrstat 56609513..56609638) refresh
 -1224> 2017-11-05 21:59:20.372619 7f23302e4700 10 mon.ceph03@2(peon).mgrstat  56609638
 -1223> 2017-11-05 21:59:20.372666 7f23302e4700 10 mon.ceph03@2(peon).mgrstat update_from_paxos v56609638 service_map e1
 -1222> 2017-11-05 21:59:20.372672 7f23302e4700 10 mon.ceph03@2(peon).mgrstat check_subs
 -1221> 2017-11-05 21:59:20.372674 7f23302e4700 20 mon.ceph03@2(peon).mgrstat update_logger
 -1220> 2017-11-05 21:59:20.372704 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(health 1..24) refresh
 -1219> 2017-11-05 21:59:20.372711 7f23302e4700 10 mon.ceph03@2(peon).health update_from_paxos
 -1218> 2017-11-05 21:59:20.372726 7f23302e4700 20 mon.ceph03@2(peon).health dump:{
    "quorum_health": {},
    "leader_health": {}
}

 -1217> 2017-11-05 21:59:20.372741 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(pgmap 56600067..56600783) post_refresh
 -1216> 2017-11-05 21:59:20.372745 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mdsmap 1..1) post_refresh
 -1215> 2017-11-05 21:59:20.372747 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(osdmap 503..1105) post_refresh
 -1214> 2017-11-05 21:59:20.372748 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(logm 55596915..55597636) post_refresh
 -1213> 2017-11-05 21:59:20.372750 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(monmap 1..14) post_refresh
 -1212> 2017-11-05 21:59:20.372752 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(auth 26252..26331) post_refresh
 -1211> 2017-11-05 21:59:20.372753 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgr 1..394) post_refresh
 -1210> 2017-11-05 21:59:20.372755 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgrstat 56609513..56609638) post_refresh
 -1209> 2017-11-05 21:59:20.372756 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(health 1..24) post_refresh
 -1208> 2017-11-05 21:59:20.372828 7f23302e4700  1 -- 10.10.1.23:6789/0 <== mon.0 10.10.1.21:6789/0 248515260 ==== paxos(lease lc 112907747 fc 112907080 pn 0 opn 0) v4 ==== 84+0+0 (618269879 0 0) 0x7f234296a700 con 0x7f2342dd4800
 -1207> 2017-11-05 21:59:20.372846 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
 -1206> 2017-11-05 21:59:20.372854 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
 -1205> 2017-11-05 21:59:20.372857 7f23302e4700 20 is_capable service=mon command= read on cap allow *
 -1204> 2017-11-05 21:59:20.372859 7f23302e4700 20  allow so far , doing grant allow *
 -1203> 2017-11-05 21:59:20.372860 7f23302e4700 20  allow all
 -1202> 2017-11-05 21:59:20.372862 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
 -1201> 2017-11-05 21:59:20.372864 7f23302e4700 20  allow so far , doing grant allow *
 -1200> 2017-11-05 21:59:20.372866 7f23302e4700 20  allow all
 -1199> 2017-11-05 21:59:20.372870 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907747) handle_lease on 112907747 now 2017-11-05 21:59:25.329563
 -1198> 2017-11-05 21:59:20.372883 7f23302e4700  1 -- 10.10.1.23:6789/0 --> 10.10.1.21:6789/0 -- paxos(lease_ack lc 112907747 fc 112907080 pn 0 opn 0) v4 -- 0x7f234296aa00 con 0
 -1197> 2017-11-05 21:59:20.372901 7f23302e4700 20 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907747) reset_lease_timeout - setting timeout event
 -1196> 2017-11-05 21:59:20.372908 7f23302e4700  5 mon.ceph03@2(peon).monmap v14 apply_mon_features wait for service to be writeable
 -1195> 2017-11-05 21:59:20.372915 7f23302e4700  5 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907747) is_readable = 1 - now=2017-11-05 21:59:20.372916 lease_expire=2017-11-05 21:59:25.329563 has v0 lc 112907747
 -1194> 2017-11-05 21:59:20.372967 7f232c2dc700 10 _calc_signature seq 28 front_crc_ = 3419319976 middle_crc = 0 data_crc = 0 sig = 17296926581295683541
 -1193> 2017-11-05 21:59:20.372984 7f232c2dc700 20 Putting signature in client message(seq # 28): sig = 17296926581295683541
 -1192> 2017-11-05 21:59:20.730805 7f232d2de700  1 -- 10.10.1.23:6789/0 >> - conn(0x7f2342de6800 :6789 s=STATE_ACCEPTING pgs=0 cs=0 l=0)._process_connection sd=37 -
 -1191> 2017-11-05 21:59:20.731044 7f232d2de700 10 mon.ceph03@2(peon) e14 ms_verify_authorizer 10.10.1.21:0/2729617137 client protocol 0
 -1190> 2017-11-05 21:59:20.731063 7f232d2de700 10 In get_auth_session_handler for protocol 0
 -1189> 2017-11-05 21:59:20.731091 7f232d2de700  2 -- 10.10.1.23:6789/0 >> 10.10.1.21:0/2729617137 conn(0x7f2342de6800 :6789 s=STATE_ACCEPTING_WAIT_SEQ pgs=2 cs=1 l=1).handle_connect_msg accept write reply msg done
 -1188> 2017-11-05 21:59:20.731217 7f232d2de700  2 -- 10.10.1.23:6789/0 >> 10.10.1.21:0/2729617137 conn(0x7f2342de6800 :6789 s=STATE_ACCEPTING_WAIT_SEQ pgs=2 cs=1 l=1)._process_connection accept get newly_acked_seq 0
 -1187> 2017-11-05 21:59:20.731270 7f232d2de700  5 -- 10.10.1.23:6789/0 >> 10.10.1.21:0/2729617137 conn(0x7f2342de6800 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=2 cs=1 l=1). rx client.? seq 1 0x7f234296d900 auth(proto 0 30 bytes epoch 0) v1
 -1186> 2017-11-05 21:59:20.731303 7f23302e4700  1 -- 10.10.1.23:6789/0 <== client.? 10.10.1.21:0/2729617137 1 ==== auth(proto 0 30 bytes epoch 0) v1 ==== 60+0+0 (673663173 0 0) 0x7f234296d900 con 0x7f2342de6800
 -1185> 2017-11-05 21:59:20.731347 7f23302e4700 10 mon.ceph03@2(peon) e14 _ms_dispatch new session 0x7f2342933180 MonSession(client.? 10.10.1.21:0/2729617137 is open ) features 0x1ffddff8eea4fffb
 -1184> 2017-11-05 21:59:20.731359 7f23302e4700 20 mon.ceph03@2(peon) e14  caps
 -1183> 2017-11-05 21:59:20.731363 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(auth 26252..26331) dispatch 0x7f234296d900 auth(proto 0 30 bytes epoch 0) v1 from client.? 10.10.1.21:0/2729617137 con 0x7f2342de6800
 -1182> 2017-11-05 21:59:20.731370 7f23302e4700  5 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907747) is_readable = 1 - now=2017-11-05 21:59:20.731371 lease_expire=2017-11-05 21:59:25.329563 has v0 lc 112907747
 -1181> 2017-11-05 21:59:20.731380 7f23302e4700 10 mon.ceph03@2(peon).auth v26331 preprocess_query auth(proto 0 30 bytes epoch 0) v1 from client.? 10.10.1.21:0/2729617137
 -1180> 2017-11-05 21:59:20.731386 7f23302e4700 10 mon.ceph03@2(peon).auth v26331 prep_auth() blob_size=30
 -1179> 2017-11-05 21:59:20.731404 7f23302e4700 10 mon.ceph03@2(peon).auth v26331 AuthMonitor::assign_global_id m=auth(proto 0 30 bytes epoch 0) v1 mon=2/3 last_allocated=55234096 max_global_id=55234096
 -1178> 2017-11-05 21:59:20.731412 7f23302e4700 10 mon.ceph03@2(peon).auth v26331 next_global_id should be 55234100
 -1177> 2017-11-05 21:59:20.731414 7f23302e4700 10 mon.ceph03@2(peon).auth v26331 not the leader, requesting more ids from leader
 -1176> 2017-11-05 21:59:20.731418 7f23302e4700  1 -- 10.10.1.23:6789/0 _send_message--> mon.0 10.10.1.21:6789/0 -- global_id  (55234096) v1 -- ?+0 0x7f2342957800
 -1175> 2017-11-05 21:59:20.731427 7f23302e4700  1 -- 10.10.1.23:6789/0 --> 10.10.1.21:6789/0 -- global_id  (55234096) v1 -- 0x7f2342957800 con 0
 -1174> 2017-11-05 21:59:20.731491 7f232c2dc700 10 _calc_signature seq 29 front_crc_ = 1755414638 middle_crc = 0 data_crc = 0 sig = 10624911126847986870
 -1173> 2017-11-05 21:59:20.731509 7f232c2dc700 20 Putting signature in client message(seq # 29): sig = 10624911126847986870
 -1172> 2017-11-05 21:59:20.732088 7f232d2de700  1 -- 10.10.1.23:6789/0 >> 10.10.1.21:0/2729617137 conn(0x7f2342de6800 :6789 s=STATE_OPEN pgs=2 cs=1 l=1).read_bulk peer close file descriptor 37
 -1171> 2017-11-05 21:59:20.732106 7f232d2de700  1 -- 10.10.1.23:6789/0 >> 10.10.1.21:0/2729617137 conn(0x7f2342de6800 :6789 s=STATE_OPEN pgs=2 cs=1 l=1).read_until read failed
 -1170> 2017-11-05 21:59:20.732123 7f232d2de700  1 -- 10.10.1.23:6789/0 >> 10.10.1.21:0/2729617137 conn(0x7f2342de6800 :6789 s=STATE_OPEN pgs=2 cs=1 l=1).process read tag failed
 -1169> 2017-11-05 21:59:20.732133 7f232d2de700  1 -- 10.10.1.23:6789/0 >> 10.10.1.21:0/2729617137 conn(0x7f2342de6800 :6789 s=STATE_OPEN pgs=2 cs=1 l=1).fault on lossy channel, failing
 -1168> 2017-11-05 21:59:20.732143 7f232d2de700  2 -- 10.10.1.23:6789/0 >> 10.10.1.21:0/2729617137 conn(0x7f2342de6800 :6789 s=STATE_OPEN pgs=2 cs=1 l=1)._stop
 -1167> 2017-11-05 21:59:20.732185 7f23302e4700 10 mon.ceph03@2(peon) e14 ms_handle_reset 0x7f2342de6800 10.10.1.21:0/2729617137
 -1166> 2017-11-05 21:59:20.732209 7f23302e4700 10 mon.ceph03@2(peon) e14 reset/close on session client.? 10.10.1.21:0/2729617137
 -1165> 2017-11-05 21:59:20.732214 7f23302e4700 10 mon.ceph03@2(peon) e14 remove_session 0x7f2342933180 client.? 10.10.1.21:0/2729617137 features 0x1ffddff8eea4fffb
 -1164> 2017-11-05 21:59:20.763005 7f232c2dc700 10 _calc_signature seq 248515261 front_crc_ = 2020420307 middle_crc = 0 data_crc = 0 sig = 16491983375567104385
 -1163> 2017-11-05 21:59:20.763020 7f232c2dc700  5 -- 10.10.1.23:6789/0 >> 10.10.1.21:6789/0 conn(0x7f2342dd4800 :-1 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=1124301 cs=1 l=0). rx mon.0 seq 248515261 0x7f234296aa00 paxos(begin lc 112907747 fc 0 pn 13700 opn 0) v4
 -1162> 2017-11-05 21:59:20.763078 7f23302e4700  1 -- 10.10.1.23:6789/0 <== mon.0 10.10.1.21:6789/0 248515261 ==== paxos(begin lc 112907747 fc 0 pn 13700 opn 0) v4 ==== 271+0+0 (2020420307 0 0) 0x7f234296aa00 con 0x7f2342dd4800
 -1161> 2017-11-05 21:59:20.763112 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
 -1160> 2017-11-05 21:59:20.763121 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
 -1159> 2017-11-05 21:59:20.763125 7f23302e4700 20 is_capable service=mon command= read on cap allow *
 -1158> 2017-11-05 21:59:20.763128 7f23302e4700 20  allow so far , doing grant allow *
 -1157> 2017-11-05 21:59:20.763129 7f23302e4700 20  allow all
 -1156> 2017-11-05 21:59:20.763131 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
 -1155> 2017-11-05 21:59:20.763133 7f23302e4700 20  allow so far , doing grant allow *
 -1154> 2017-11-05 21:59:20.763134 7f23302e4700 20  allow all
 -1153> 2017-11-05 21:59:20.763139 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907747) handle_begin paxos(begin lc 112907747 fc 0 pn 13700 opn 0) v4
 -1152> 2017-11-05 21:59:20.763143 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos updating c 112907080..112907747) accepting value for 112907748 pn 13700
 -1151> 2017-11-05 21:59:20.804042 7f23302e4700  1 -- 10.10.1.23:6789/0 --> 10.10.1.21:6789/0 -- paxos(accept lc 112907747 fc 0 pn 13700 opn 0) v4 -- 0x7f234296a700 con 0
 -1150> 2017-11-05 21:59:20.804165 7f232c2dc700 10 _calc_signature seq 30 front_crc_ = 4077125949 middle_crc = 0 data_crc = 0 sig = 2958054525900925623
 -1149> 2017-11-05 21:59:20.804183 7f232c2dc700 20 Putting signature in client message(seq # 30): sig = 2958054525900925623
 -1148> 2017-11-05 21:59:20.840669 7f232c2dc700 10 _calc_signature seq 248515262 front_crc_ = 2514305494 middle_crc = 0 data_crc = 0 sig = 3250237394762425023
 -1147> 2017-11-05 21:59:20.840684 7f232c2dc700  5 -- 10.10.1.23:6789/0 >> 10.10.1.21:6789/0 conn(0x7f2342dd4800 :-1 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=1124301 cs=1 l=0). rx mon.0 seq 248515262 0x7f234296a700 paxos(commit lc 112907748 fc 0 pn 13700 opn 0) v4
 -1146> 2017-11-05 21:59:20.840743 7f23302e4700  1 -- 10.10.1.23:6789/0 <== mon.0 10.10.1.21:6789/0 248515262 ==== paxos(commit lc 112907748 fc 0 pn 13700 opn 0) v4 ==== 271+0+0 (2514305494 0 0) 0x7f234296a700 con 0x7f2342dd4800
 -1145> 2017-11-05 21:59:20.840777 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
 -1144> 2017-11-05 21:59:20.840789 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
 -1143> 2017-11-05 21:59:20.840793 7f23302e4700 20 is_capable service=mon command= read on cap allow *
 -1142> 2017-11-05 21:59:20.840796 7f23302e4700 20  allow so far , doing grant allow *
 -1141> 2017-11-05 21:59:20.840797 7f23302e4700 20  allow all
 -1140> 2017-11-05 21:59:20.840799 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
 -1139> 2017-11-05 21:59:20.840801 7f23302e4700 20  allow so far , doing grant allow *
 -1138> 2017-11-05 21:59:20.840802 7f23302e4700 20  allow all
 -1137> 2017-11-05 21:59:20.840807 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos updating c 112907080..112907747) handle_commit on 112907748
 -1136> 2017-11-05 21:59:20.840831 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos updating c 112907080..112907748) store_state [112907748..112907748]
 -1135> 2017-11-05 21:59:20.840896 7f232c2dc700 10 _calc_signature seq 248515263 front_crc_ = 406934426 middle_crc = 0 data_crc = 0 sig = 15435441924216793349
 -1134> 2017-11-05 21:59:20.840907 7f232c2dc700  5 -- 10.10.1.23:6789/0 >> 10.10.1.21:6789/0 conn(0x7f2342dd4800 :-1 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=1124301 cs=1 l=0). rx mon.0 seq 248515263 0x7f234296bc00 paxos(lease lc 112907748 fc 112907080 pn 0 opn 0) v4
 -1133> 2017-11-05 21:59:20.881717 7f23302e4700 10 mon.ceph03@2(peon) e14 refresh_from_paxos
 -1132> 2017-11-05 21:59:20.881751 7f23302e4700 10 mon.ceph03@2(peon) e14 refresh_from_paxos no cluster_fingerprint
 -1131> 2017-11-05 21:59:20.881787 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(pgmap 56600067..56600783) refresh
 -1130> 2017-11-05 21:59:20.881833 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mdsmap 1..1) refresh
 -1129> 2017-11-05 21:59:20.881879 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(osdmap 503..1105) refresh
 -1128> 2017-11-05 21:59:20.881912 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(logm 55596915..55597636) refresh
 -1127> 2017-11-05 21:59:20.881918 7f23302e4700 10 mon.ceph03@2(peon).log v55597636 update_from_paxos
 -1126> 2017-11-05 21:59:20.881921 7f23302e4700 10 mon.ceph03@2(peon).log v55597636 update_from_paxos version 55597636 summary v 55597636
 -1125> 2017-11-05 21:59:20.881960 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(monmap 1..14) refresh
 -1124> 2017-11-05 21:59:20.881984 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(auth 26252..26332) refresh
 -1123> 2017-11-05 21:59:20.881990 7f23302e4700 10 mon.ceph03@2(peon).auth v26332 update_from_paxos
 -1122> 2017-11-05 21:59:20.882006 7f23302e4700 10 mon.ceph03@2(peon).auth v26332 update_from_paxos version 26332 keys ver 26331 latest 26321
 -1121> 2017-11-05 21:59:20.882015 7f23302e4700 10 mon.ceph03@2(peon).auth v26332 update_from_paxos key server version 26331
 -1120> 2017-11-05 21:59:20.882024 7f23302e4700 20 mon.ceph03@2(peon).auth v26332 update_from_paxos walking through version 26332 len 19
 -1119> 2017-11-05 21:59:20.882028 7f23302e4700 10 mon.ceph03@2(peon).auth v26332 update_from_paxos() last_allocated_id=55234096 max_global_id=55244096 format_version 2
 -1118> 2017-11-05 21:59:20.882068 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgr 1..394) refresh
 -1117> 2017-11-05 21:59:20.882081 7f23302e4700  4 mgrc handle_mgr_map Got map version 394
 -1116> 2017-11-05 21:59:20.882084 7f23302e4700  4 mgrc handle_mgr_map Active mgr is now 10.10.1.21:6806/22556
 -1115> 2017-11-05 21:59:20.882125 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgrstat 56609513..56609638) refresh
 -1114> 2017-11-05 21:59:20.882133 7f23302e4700 10 mon.ceph03@2(peon).mgrstat  56609638
 -1113> 2017-11-05 21:59:20.882181 7f23302e4700 10 mon.ceph03@2(peon).mgrstat update_from_paxos v56609638 service_map e1
 -1112> 2017-11-05 21:59:20.882187 7f23302e4700 10 mon.ceph03@2(peon).mgrstat check_subs
 -1111> 2017-11-05 21:59:20.882189 7f23302e4700 20 mon.ceph03@2(peon).mgrstat update_logger
 -1110> 2017-11-05 21:59:20.882221 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(health 1..24) refresh
 -1109> 2017-11-05 21:59:20.882228 7f23302e4700 10 mon.ceph03@2(peon).health update_from_paxos
 -1108> 2017-11-05 21:59:20.882244 7f23302e4700 20 mon.ceph03@2(peon).health dump:{
    "quorum_health": {},
    "leader_health": {}
}

 -1107> 2017-11-05 21:59:20.882259 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(pgmap 56600067..56600783) post_refresh
 -1106> 2017-11-05 21:59:20.882261 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mdsmap 1..1) post_refresh
 -1105> 2017-11-05 21:59:20.882263 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(osdmap 503..1105) post_refresh
 -1104> 2017-11-05 21:59:20.882265 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(logm 55596915..55597636) post_refresh
 -1103> 2017-11-05 21:59:20.882267 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(monmap 1..14) post_refresh
 -1102> 2017-11-05 21:59:20.882269 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(auth 26252..26332) post_refresh
 -1101> 2017-11-05 21:59:20.882273 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(auth 26252..26332) dispatch 0x7f234296d900 auth(proto 0 30 bytes epoch 0) v1 from client.? 10.10.1.21:0/2729617137 con 0x7f2342de6800
 -1100> 2017-11-05 21:59:20.882299 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(auth 26252..26332)  discarding message from disconnected client client.? 10.10.1.21:0/2729617137 auth(proto 0 30 bytes epoch 0) v1
 -1099> 2017-11-05 21:59:20.882321 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgr 1..394) post_refresh
 -1098> 2017-11-05 21:59:20.882327 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgrstat 56609513..56609638) post_refresh
 -1097> 2017-11-05 21:59:20.882329 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(health 1..24) post_refresh
 -1096> 2017-11-05 21:59:20.882401 7f23302e4700  1 -- 10.10.1.23:6789/0 <== mon.0 10.10.1.21:6789/0 248515263 ==== paxos(lease lc 112907748 fc 112907080 pn 0 opn 0) v4 ==== 84+0+0 (406934426 0 0) 0x7f234296bc00 con 0x7f2342dd4800
 -1095> 2017-11-05 21:59:20.882417 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
 -1094> 2017-11-05 21:59:20.882425 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
 -1093> 2017-11-05 21:59:20.882429 7f23302e4700 20 is_capable service=mon command= read on cap allow *
 -1092> 2017-11-05 21:59:20.882431 7f23302e4700 20  allow so far , doing grant allow *
 -1091> 2017-11-05 21:59:20.882433 7f23302e4700 20  allow all
 -1090> 2017-11-05 21:59:20.882435 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
 -1089> 2017-11-05 21:59:20.882436 7f23302e4700 20  allow so far , doing grant allow *
 -1088> 2017-11-05 21:59:20.882437 7f23302e4700 20  allow all
 -1087> 2017-11-05 21:59:20.882441 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907748) handle_lease on 112907748 now 2017-11-05 21:59:25.839092
 -1086> 2017-11-05 21:59:20.882458 7f23302e4700  1 -- 10.10.1.23:6789/0 --> 10.10.1.21:6789/0 -- paxos(lease_ack lc 112907748 fc 112907080 pn 0 opn 0) v4 -- 0x7f234296a700 con 0
 -1085> 2017-11-05 21:59:20.882475 7f23302e4700 20 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907748) reset_lease_timeout - setting timeout event
 -1084> 2017-11-05 21:59:20.882481 7f23302e4700  5 mon.ceph03@2(peon).monmap v14 apply_mon_features wait for service to be writeable
 -1083> 2017-11-05 21:59:20.882484 7f23302e4700  5 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907748) is_readable = 1 - now=2017-11-05 21:59:20.882484 lease_expire=2017-11-05 21:59:25.839092 has v0 lc 112907748
 -1082> 2017-11-05 21:59:20.882541 7f232c2dc700 10 _calc_signature seq 31 front_crc_ = 235397502 middle_crc = 0 data_crc = 0 sig = 8528813682334572839
 -1081> 2017-11-05 21:59:20.882558 7f232c2dc700 20 Putting signature in client message(seq # 31): sig = 8528813682334572839
 -1080> 2017-11-05 21:59:21.870644 7f232c2dc700 10 _calc_signature seq 248515264 front_crc_ = 791506274 middle_crc = 0 data_crc = 0 sig = 13748375555303897324
 -1079> 2017-11-05 21:59:21.870666 7f232c2dc700  5 -- 10.10.1.23:6789/0 >> 10.10.1.21:6789/0 conn(0x7f2342dd4800 :-1 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=1124301 cs=1 l=0). rx mon.0 seq 248515264 0x7f234296a700 paxos(begin lc 112907748 fc 0 pn 13700 opn 0) v4
 -1078> 2017-11-05 21:59:21.870713 7f23302e4700  1 -- 10.10.1.23:6789/0 <== mon.0 10.10.1.21:6789/0 248515264 ==== paxos(begin lc 112907748 fc 0 pn 13700 opn 0) v4 ==== 10594+0+0 (791506274 0 0) 0x7f234296a700 con 0x7f2342dd4800
 -1077> 2017-11-05 21:59:21.870752 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
 -1076> 2017-11-05 21:59:21.870763 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
 -1075> 2017-11-05 21:59:21.870767 7f23302e4700 20 is_capable service=mon command= read on cap allow *
 -1074> 2017-11-05 21:59:21.870771 7f23302e4700 20  allow so far , doing grant allow *
 -1073> 2017-11-05 21:59:21.870772 7f23302e4700 20  allow all
 -1072> 2017-11-05 21:59:21.870775 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
 -1071> 2017-11-05 21:59:21.870776 7f23302e4700 20  allow so far , doing grant allow *
 -1070> 2017-11-05 21:59:21.870778 7f23302e4700 20  allow all
 -1069> 2017-11-05 21:59:21.870785 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907748) handle_begin paxos(begin lc 112907748 fc 0 pn 13700 opn 0) v4
 -1068> 2017-11-05 21:59:21.870790 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos updating c 112907080..112907748) accepting value for 112907749 pn 13700
 -1067> 2017-11-05 21:59:21.911438 7f23302e4700  1 -- 10.10.1.23:6789/0 --> 10.10.1.21:6789/0 -- paxos(accept lc 112907748 fc 0 pn 13700 opn 0) v4 -- 0x7f234296bc00 con 0
 -1066> 2017-11-05 21:59:21.911518 7f232c2dc700 10 _calc_signature seq 32 front_crc_ = 3333053517 middle_crc = 0 data_crc = 0 sig = 8416660674998655820
 -1065> 2017-11-05 21:59:21.911534 7f232c2dc700 20 Putting signature in client message(seq # 32): sig = 8416660674998655820
 -1064> 2017-11-05 21:59:21.949630 7f232c2dc700 10 _calc_signature seq 248515265 front_crc_ = 370334524 middle_crc = 0 data_crc = 0 sig = 5896854534053460549
 -1063> 2017-11-05 21:59:21.949647 7f232c2dc700  5 -- 10.10.1.23:6789/0 >> 10.10.1.21:6789/0 conn(0x7f2342dd4800 :-1 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=1124301 cs=1 l=0). rx mon.0 seq 248515265 0x7f234296bc00 paxos(commit lc 112907749 fc 0 pn 13700 opn 0) v4
 -1062> 2017-11-05 21:59:21.949684 7f23302e4700  1 -- 10.10.1.23:6789/0 <== mon.0 10.10.1.21:6789/0 248515265 ==== paxos(commit lc 112907749 fc 0 pn 13700 opn 0) v4 ==== 10594+0+0 (370334524 0 0) 0x7f234296bc00 con 0x7f2342dd4800
 -1061> 2017-11-05 21:59:21.949717 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
 -1060> 2017-11-05 21:59:21.949727 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
 -1059> 2017-11-05 21:59:21.949733 7f23302e4700 20 is_capable service=mon command= read on cap allow *
 -1058> 2017-11-05 21:59:21.949736 7f23302e4700 20  allow so far , doing grant allow *
 -1057> 2017-11-05 21:59:21.949738 7f23302e4700 20  allow all
 -1056> 2017-11-05 21:59:21.949740 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
 -1055> 2017-11-05 21:59:21.949741 7f23302e4700 20  allow so far , doing grant allow *
 -1054> 2017-11-05 21:59:21.949744 7f23302e4700 20  allow all
 -1053> 2017-11-05 21:59:21.949769 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos updating c 112907080..112907748) handle_commit on 112907749
 -1052> 2017-11-05 21:59:21.949778 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos updating c 112907080..112907749) store_state [112907749..112907749]
 -1051> 2017-11-05 21:59:21.950003 7f232c2dc700 10 _calc_signature seq 248515266 front_crc_ = 2497589639 middle_crc = 0 data_crc = 0 sig = 8276221671505461764
 -1050> 2017-11-05 21:59:21.950014 7f232c2dc700  5 -- 10.10.1.23:6789/0 >> 10.10.1.21:6789/0 conn(0x7f2342dd4800 :-1 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=1124301 cs=1 l=0). rx mon.0 seq 248515266 0x7f234296b900 paxos(lease lc 112907749 fc 112907080 pn 0 opn 0) v4
 -1049> 2017-11-05 21:59:21.989093 7f23302e4700 10 mon.ceph03@2(peon) e14 refresh_from_paxos
 -1048> 2017-11-05 21:59:21.989125 7f23302e4700 10 mon.ceph03@2(peon) e14 refresh_from_paxos no cluster_fingerprint
 -1047> 2017-11-05 21:59:21.989158 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(pgmap 56600067..56600783) refresh
 -1046> 2017-11-05 21:59:21.989203 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mdsmap 1..1) refresh
 -1045> 2017-11-05 21:59:21.989246 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(osdmap 503..1105) refresh
 -1044> 2017-11-05 21:59:21.989277 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(logm 55596915..55597637) refresh
 -1043> 2017-11-05 21:59:21.989284 7f23302e4700 10 mon.ceph03@2(peon).log v55597637 update_from_paxos
 -1042> 2017-11-05 21:59:21.989286 7f23302e4700 10 mon.ceph03@2(peon).log v55597637 update_from_paxos version 55597637 summary v 55597636
 -1041> 2017-11-05 21:59:21.989294 7f23302e4700 10 mon.ceph03@2(peon).log v55597637 update_from_paxos latest full 55597636
 -1040> 2017-11-05 21:59:21.989312 7f23302e4700  7 mon.ceph03@2(peon).log v55597637 update_from_paxos applying incremental log 55597637 2017-11-05 21:59:20.718439 mgr.ceph01mgr client.54805773 10.10.1.21:0/1619113657 1818 : audit [DBG] from='client.55026405 10.10.1.21:0/3042078038' entity='client.admin' cmd=[{"prefix": "osd pool stats", "target": ["mgr", ""], "format": "json"}]: dispatch
 -1039> 2017-11-05 21:59:21.989333 7f23302e4700 20 mon.ceph03@2(peon).log v55597637 update_from_paxos logging for channel 'audit' to file '/var/log/ceph/ceph.audit.log'
 -1038> 2017-11-05 21:59:21.989342 7f23302e4700  7 mon.ceph03@2(peon).log v55597637 update_from_paxos applying incremental log 55597637 2017-11-05 21:59:20.943050 mon.ceph01mon mon.0 10.10.1.21:6789/0 865 : audit [DBG] from='client.? 10.10.1.21:0/2729617137' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch
 -1037> 2017-11-05 21:59:21.989352 7f23302e4700 20 mon.ceph03@2(peon).log v55597637 update_from_paxos logging for channel 'audit' to file '/var/log/ceph/ceph.audit.log'
 -1036> 2017-11-05 21:59:21.989358 7f23302e4700  7 mon.ceph03@2(peon).log v55597637 update_from_paxos applying incremental log 55597637 2017-11-05 21:59:20.967452 mon.ceph02mon mon.1 10.10.1.22:6789/0 362 : audit [DBG] from='client.? 10.10.1.21:0/1655216627' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch
 -1035> 2017-11-05 21:59:21.989364 7f23302e4700 20 mon.ceph03@2(peon).log v55597637 update_from_paxos logging for channel 'audit' to file '/var/log/ceph/ceph.audit.log'
 -1034> 2017-11-05 21:59:21.989369 7f23302e4700  7 mon.ceph03@2(peon).log v55597637 update_from_paxos applying incremental log 55597637 2017-11-05 21:59:20.999972 mon.ceph01mon mon.0 10.10.1.21:6789/0 866 : audit [DBG] from='client.? 10.10.1.21:0/1886445839' entity='client.admin' cmd=[{"prefix": "df", "format": "json"}]: dispatch
 -1033> 2017-11-05 21:59:21.989374 7f23302e4700 20 mon.ceph03@2(peon).log v55597637 update_from_paxos logging for channel 'audit' to file '/var/log/ceph/ceph.audit.log'
 -1032> 2017-11-05 21:59:21.989394 7f23302e4700 15 mon.ceph03@2(peon).log v55597637 update_from_paxos logging for 0 channels
 -1031> 2017-11-05 21:59:21.989400 7f23302e4700 10 mon.ceph03@2(peon).log v55597637 check_subs
 -1030> 2017-11-05 21:59:21.989443 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(monmap 1..14) refresh
 -1029> 2017-11-05 21:59:21.989466 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(auth 26252..26332) refresh
 -1028> 2017-11-05 21:59:21.989473 7f23302e4700 10 mon.ceph03@2(peon).auth v26332 update_from_paxos
 -1027> 2017-11-05 21:59:21.989511 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgr 1..394) refresh
 -1026> 2017-11-05 21:59:21.989524 7f23302e4700  4 mgrc handle_mgr_map Got map version 394
 -1025> 2017-11-05 21:59:21.989528 7f23302e4700  4 mgrc handle_mgr_map Active mgr is now 10.10.1.21:6806/22556
 -1024> 2017-11-05 21:59:21.989563 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgrstat 56609513..56609638) refresh
 -1023> 2017-11-05 21:59:21.989569 7f23302e4700 10 mon.ceph03@2(peon).mgrstat  56609638
 -1022> 2017-11-05 21:59:21.989611 7f23302e4700 10 mon.ceph03@2(peon).mgrstat update_from_paxos v56609638 service_map e1
 -1021> 2017-11-05 21:59:21.989617 7f23302e4700 10 mon.ceph03@2(peon).mgrstat check_subs
 -1020> 2017-11-05 21:59:21.989619 7f23302e4700 20 mon.ceph03@2(peon).mgrstat update_logger
 -1019> 2017-11-05 21:59:21.989649 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(health 1..24) refresh
 -1018> 2017-11-05 21:59:21.989656 7f23302e4700 10 mon.ceph03@2(peon).health update_from_paxos
 -1017> 2017-11-05 21:59:21.989671 7f23302e4700 20 mon.ceph03@2(peon).health dump:{
    "quorum_health": {},
    "leader_health": {}
}

 -1016> 2017-11-05 21:59:21.989686 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(pgmap 56600067..56600783) post_refresh
 -1015> 2017-11-05 21:59:21.989688 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mdsmap 1..1) post_refresh
 -1014> 2017-11-05 21:59:21.989690 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(osdmap 503..1105) post_refresh
 -1013> 2017-11-05 21:59:21.989692 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(logm 55596915..55597637) post_refresh
 -1012> 2017-11-05 21:59:21.989693 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(monmap 1..14) post_refresh
 -1011> 2017-11-05 21:59:21.989695 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(auth 26252..26332) post_refresh
 -1010> 2017-11-05 21:59:21.989696 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgr 1..394) post_refresh
 -1009> 2017-11-05 21:59:21.989700 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgrstat 56609513..56609638) post_refresh
 -1008> 2017-11-05 21:59:21.989701 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(health 1..24) post_refresh
 -1007> 2017-11-05 21:59:21.989772 7f23302e4700  1 -- 10.10.1.23:6789/0 <== mon.0 10.10.1.21:6789/0 248515266 ==== paxos(lease lc 112907749 fc 112907080 pn 0 opn 0) v4 ==== 84+0+0 (2497589639 0 0) 0x7f234296b900 con 0x7f2342dd4800
 -1006> 2017-11-05 21:59:21.989790 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
 -1005> 2017-11-05 21:59:21.989799 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
 -1004> 2017-11-05 21:59:21.989803 7f23302e4700 20 is_capable service=mon command= read on cap allow *
 -1003> 2017-11-05 21:59:21.989806 7f23302e4700 20  allow so far , doing grant allow *
 -1002> 2017-11-05 21:59:21.989808 7f23302e4700 20  allow all
 -1001> 2017-11-05 21:59:21.989810 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
 -1000> 2017-11-05 21:59:21.989811 7f23302e4700 20  allow so far , doing grant allow *
  -999> 2017-11-05 21:59:21.989813 7f23302e4700 20  allow all
  -998> 2017-11-05 21:59:21.989819 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907749) handle_lease on 112907749 now 2017-11-05 21:59:26.947993
  -997> 2017-11-05 21:59:21.989833 7f23302e4700  1 -- 10.10.1.23:6789/0 --> 10.10.1.21:6789/0 -- paxos(lease_ack lc 112907749 fc 112907080 pn 0 opn 0) v4 -- 0x7f234296bc00 con 0
  -996> 2017-11-05 21:59:21.989851 7f23302e4700 20 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907749) reset_lease_timeout - setting timeout event
  -995> 2017-11-05 21:59:21.989863 7f23302e4700  5 mon.ceph03@2(peon).monmap v14 apply_mon_features wait for service to be writeable
  -994> 2017-11-05 21:59:21.989866 7f23302e4700  5 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907749) is_readable = 1 - now=2017-11-05 21:59:21.989866 lease_expire=2017-11-05 21:59:26.947993 has v0 lc 112907749
  -993> 2017-11-05 21:59:21.989898 7f232c2dc700 10 _calc_signature seq 33 front_crc_ = 1097711905 middle_crc = 0 data_crc = 0 sig = 17535115145835410118
  -992> 2017-11-05 21:59:21.989913 7f232c2dc700 20 Putting signature in client message(seq # 33): sig = 17535115145835410118
  -991> 2017-11-05 21:59:22.692236 7f2332ae9700 11 mon.ceph03@2(peon) e14 tick
  -990> 2017-11-05 21:59:22.692259 7f2332ae9700 10 mon.ceph03@2(peon).mds e1
  -989> 2017-11-05 21:59:22.692264 7f2332ae9700 10 mon.ceph03@2(peon).osd e1105 e1105: 9 total, 9 up, 9 in
  -988> 2017-11-05 21:59:22.692268 7f2332ae9700 10 mon.ceph03@2(peon).log v55597637 log
  -987> 2017-11-05 21:59:22.692271 7f2332ae9700 10 mon.ceph03@2(peon).auth v26332 auth
  -986> 2017-11-05 21:59:22.692275 7f2332ae9700 10 mon.ceph03@2(peon).health tick
  -985> 2017-11-05 21:59:22.692278 7f2332ae9700 20 mon.ceph03@2(peon).health check_member_health
  -984> 2017-11-05 21:59:22.692359 7f2332ae9700 10 mon.ceph03@2(peon).health check_member_health avail 92% total 277 GB, used 5676 MB, avail 257 GB
  -983> 2017-11-05 21:59:22.692371 7f2332ae9700 20 mon.ceph03@2(peon) e14 sync_trim_providers
  -982> 2017-11-05 21:59:22.898359 7f232f2e2700  1 -- 10.10.1.23:0/15195 --> 10.10.1.21:6806/22556 -- mgrreport(unknown.ceph03 +0-0 packed 1734) v4 -- 0x7f2342beb8c0 con 0
  -981> 2017-11-05 21:59:22.898492 7f232d2de700 10 _calc_signature seq 4 front_crc_ = 2516153275 middle_crc = 0 data_crc = 0 sig = 10573281299783805524
  -980> 2017-11-05 21:59:22.898512 7f232d2de700 20 Putting signature in client message(seq # 4): sig = 10573281299783805524
  -979> 2017-11-05 21:59:22.990920 7f232c2dc700 10 _calc_signature seq 248515267 front_crc_ = 2763070699 middle_crc = 0 data_crc = 0 sig = 14556592154937786364
  -978> 2017-11-05 21:59:22.990939 7f232c2dc700  5 -- 10.10.1.23:6789/0 >> 10.10.1.21:6789/0 conn(0x7f2342dd4800 :-1 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=1124301 cs=1 l=0). rx mon.0 seq 248515267 0x7f234296bc00 paxos(begin lc 112907749 fc 0 pn 13700 opn 0) v4
  -977> 2017-11-05 21:59:22.990990 7f23302e4700  1 -- 10.10.1.23:6789/0 <== mon.0 10.10.1.21:6789/0 248515267 ==== paxos(begin lc 112907749 fc 0 pn 13700 opn 0) v4 ==== 3352+0+0 (2763070699 0 0) 0x7f234296bc00 con 0x7f2342dd4800
  -976> 2017-11-05 21:59:22.991024 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
  -975> 2017-11-05 21:59:22.991034 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
  -974> 2017-11-05 21:59:22.991038 7f23302e4700 20 is_capable service=mon command= read on cap allow *
  -973> 2017-11-05 21:59:22.991042 7f23302e4700 20  allow so far , doing grant allow *
  -972> 2017-11-05 21:59:22.991043 7f23302e4700 20  allow all
  -971> 2017-11-05 21:59:22.991047 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
  -970> 2017-11-05 21:59:22.991048 7f23302e4700 20  allow so far , doing grant allow *
  -969> 2017-11-05 21:59:22.991049 7f23302e4700 20  allow all
  -968> 2017-11-05 21:59:22.991057 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907749) handle_begin paxos(begin lc 112907749 fc 0 pn 13700 opn 0) v4
  -967> 2017-11-05 21:59:22.991061 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos updating c 112907080..112907749) accepting value for 112907750 pn 13700
  -966> 2017-11-05 21:59:23.029839 7f23302e4700  1 -- 10.10.1.23:6789/0 --> 10.10.1.21:6789/0 -- paxos(accept lc 112907749 fc 0 pn 13700 opn 0) v4 -- 0x7f234296b900 con 0
  -965> 2017-11-05 21:59:23.029939 7f232c2dc700 10 _calc_signature seq 34 front_crc_ = 2366506105 middle_crc = 0 data_crc = 0 sig = 4827263851779999549
  -964> 2017-11-05 21:59:23.029956 7f232c2dc700 20 Putting signature in client message(seq # 34): sig = 4827263851779999549
  -963> 2017-11-05 21:59:23.068457 7f232c2dc700 10 _calc_signature seq 248515268 front_crc_ = 621179324 middle_crc = 0 data_crc = 0 sig = 10355367633632527010
  -962> 2017-11-05 21:59:23.068476 7f232c2dc700  5 -- 10.10.1.23:6789/0 >> 10.10.1.21:6789/0 conn(0x7f2342dd4800 :-1 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=1124301 cs=1 l=0). rx mon.0 seq 248515268 0x7f234296b900 paxos(commit lc 112907750 fc 0 pn 13700 opn 0) v4
  -961> 2017-11-05 21:59:23.068513 7f23302e4700  1 -- 10.10.1.23:6789/0 <== mon.0 10.10.1.21:6789/0 248515268 ==== paxos(commit lc 112907750 fc 0 pn 13700 opn 0) v4 ==== 3352+0+0 (621179324 0 0) 0x7f234296b900 con 0x7f2342dd4800
  -960> 2017-11-05 21:59:23.068555 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
  -959> 2017-11-05 21:59:23.068564 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
  -958> 2017-11-05 21:59:23.068573 7f23302e4700 20 is_capable service=mon command= read on cap allow *
  -957> 2017-11-05 21:59:23.068576 7f23302e4700 20  allow so far , doing grant allow *
  -956> 2017-11-05 21:59:23.068577 7f23302e4700 20  allow all
  -955> 2017-11-05 21:59:23.068579 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
  -954> 2017-11-05 21:59:23.068580 7f23302e4700 20  allow so far , doing grant allow *
  -953> 2017-11-05 21:59:23.068582 7f23302e4700 20  allow all
  -952> 2017-11-05 21:59:23.068587 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos updating c 112907080..112907749) handle_commit on 112907750
  -951> 2017-11-05 21:59:23.068593 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos updating c 112907080..112907750) store_state [112907750..112907750]
  -950> 2017-11-05 21:59:23.068805 7f232c2dc700 10 _calc_signature seq 248515269 front_crc_ = 2140618806 middle_crc = 0 data_crc = 0 sig = 6581431197576721299
  -949> 2017-11-05 21:59:23.068837 7f232c2dc700  5 -- 10.10.1.23:6789/0 >> 10.10.1.21:6789/0 conn(0x7f2342dd4800 :-1 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=1124301 cs=1 l=0). rx mon.0 seq 248515269 0x7f234296b600 paxos(lease lc 112907750 fc 112907080 pn 0 opn 0) v4
  -948> 2017-11-05 21:59:23.107515 7f23302e4700 10 mon.ceph03@2(peon) e14 refresh_from_paxos
  -947> 2017-11-05 21:59:23.107551 7f23302e4700 10 mon.ceph03@2(peon) e14 refresh_from_paxos no cluster_fingerprint
  -946> 2017-11-05 21:59:23.107591 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(pgmap 56600067..56600783) refresh
  -945> 2017-11-05 21:59:23.107645 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mdsmap 1..1) refresh
  -944> 2017-11-05 21:59:23.107695 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(osdmap 503..1105) refresh
  -943> 2017-11-05 21:59:23.107731 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(logm 55596915..55597637) refresh
  -942> 2017-11-05 21:59:23.107738 7f23302e4700 10 mon.ceph03@2(peon).log v55597637 update_from_paxos
  -941> 2017-11-05 21:59:23.107741 7f23302e4700 10 mon.ceph03@2(peon).log v55597637 update_from_paxos version 55597637 summary v 55597637
  -940> 2017-11-05 21:59:23.107783 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(monmap 1..14) refresh
  -939> 2017-11-05 21:59:23.107810 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(auth 26252..26332) refresh
  -938> 2017-11-05 21:59:23.107817 7f23302e4700 10 mon.ceph03@2(peon).auth v26332 update_from_paxos
  -937> 2017-11-05 21:59:23.107860 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgr 1..394) refresh
  -936> 2017-11-05 21:59:23.107875 7f23302e4700  4 mgrc handle_mgr_map Got map version 394
  -935> 2017-11-05 21:59:23.107878 7f23302e4700  4 mgrc handle_mgr_map Active mgr is now 10.10.1.21:6806/22556
  -934> 2017-11-05 21:59:23.107922 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgrstat 56609513..56609639) refresh
  -933> 2017-11-05 21:59:23.107930 7f23302e4700 10 mon.ceph03@2(peon).mgrstat  56609639
  -932> 2017-11-05 21:59:23.107981 7f23302e4700 10 mon.ceph03@2(peon).mgrstat update_from_paxos v56609639 service_map e1
  -931> 2017-11-05 21:59:23.107987 7f23302e4700 10 mon.ceph03@2(peon).mgrstat check_subs
  -930> 2017-11-05 21:59:23.107989 7f23302e4700 20 mon.ceph03@2(peon).mgrstat update_logger
  -929> 2017-11-05 21:59:23.108023 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(health 1..24) refresh
  -928> 2017-11-05 21:59:23.108031 7f23302e4700 10 mon.ceph03@2(peon).health update_from_paxos
  -927> 2017-11-05 21:59:23.108048 7f23302e4700 20 mon.ceph03@2(peon).health dump:{
    "quorum_health": {},
    "leader_health": {}
}

  -926> 2017-11-05 21:59:23.108064 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(pgmap 56600067..56600783) post_refresh
  -925> 2017-11-05 21:59:23.108067 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mdsmap 1..1) post_refresh
  -924> 2017-11-05 21:59:23.108068 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(osdmap 503..1105) post_refresh
  -923> 2017-11-05 21:59:23.108070 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(logm 55596915..55597637) post_refresh
  -922> 2017-11-05 21:59:23.108072 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(monmap 1..14) post_refresh
  -921> 2017-11-05 21:59:23.108074 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(auth 26252..26332) post_refresh
  -920> 2017-11-05 21:59:23.108075 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgr 1..394) post_refresh
  -919> 2017-11-05 21:59:23.108092 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgrstat 56609513..56609639) post_refresh
  -918> 2017-11-05 21:59:23.108094 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(health 1..24) post_refresh
  -917> 2017-11-05 21:59:23.108202 7f23302e4700  1 -- 10.10.1.23:6789/0 <== mon.0 10.10.1.21:6789/0 248515269 ==== paxos(lease lc 112907750 fc 112907080 pn 0 opn 0) v4 ==== 84+0+0 (2140618806 0 0) 0x7f234296b600 con 0x7f2342dd4800
  -916> 2017-11-05 21:59:23.108223 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
  -915> 2017-11-05 21:59:23.108232 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
  -914> 2017-11-05 21:59:23.108247 7f23302e4700 20 is_capable service=mon command= read on cap allow *
  -913> 2017-11-05 21:59:23.108253 7f23302e4700 20  allow so far , doing grant allow *
  -912> 2017-11-05 21:59:23.108255 7f23302e4700 20  allow all
  -911> 2017-11-05 21:59:23.108257 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
  -910> 2017-11-05 21:59:23.108259 7f23302e4700 20  allow so far , doing grant allow *
  -909> 2017-11-05 21:59:23.108260 7f23302e4700 20  allow all
  -908> 2017-11-05 21:59:23.108265 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907750) handle_lease on 112907750 now 2017-11-05 21:59:28.066966
  -907> 2017-11-05 21:59:23.108279 7f23302e4700  1 -- 10.10.1.23:6789/0 --> 10.10.1.21:6789/0 -- paxos(lease_ack lc 112907750 fc 112907080 pn 0 opn 0) v4 -- 0x7f234296b900 con 0
  -906> 2017-11-05 21:59:23.108297 7f23302e4700 20 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907750) reset_lease_timeout - setting timeout event
  -905> 2017-11-05 21:59:23.108308 7f23302e4700  5 mon.ceph03@2(peon).monmap v14 apply_mon_features wait for service to be writeable
  -904> 2017-11-05 21:59:23.108312 7f23302e4700  5 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907750) is_readable = 1 - now=2017-11-05 21:59:23.108313 lease_expire=2017-11-05 21:59:28.066966 has v0 lc 112907750
  -903> 2017-11-05 21:59:23.108370 7f232c2dc700 10 _calc_signature seq 35 front_crc_ = 2829686340 middle_crc = 0 data_crc = 0 sig = 14457767592524038708
  -902> 2017-11-05 21:59:23.108388 7f232c2dc700 20 Putting signature in client message(seq # 35): sig = 14457767592524038708
  -901> 2017-11-05 21:59:23.247750 7f232cadd700 10 _calc_signature seq 123007633 front_crc_ = 992124289 middle_crc = 0 data_crc = 0 sig = 9589994617269070540
  -900> 2017-11-05 21:59:23.247777 7f232cadd700  5 -- 10.10.1.23:6789/0 >> 10.10.1.22:6789/0 conn(0x7f2342dd3000 :-1 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=973717 cs=1 l=0). rx mon.1 seq 123007633 0x7f234298df80 mon_health( service 1 op tell e 0 r 0 ) v1
  -899> 2017-11-05 21:59:23.247839 7f23302e4700  1 -- 10.10.1.23:6789/0 <== mon.1 10.10.1.22:6789/0 123007633 ==== mon_health( service 1 op tell e 0 r 0 ) v1 ==== 108+0+0 (992124289 0 0) 0x7f234298df80 con 0x7f2342dd3000
  -898> 2017-11-05 21:59:23.247874 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930c00 for mon.1 10.10.1.22:6789/0
  -897> 2017-11-05 21:59:23.247884 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
  -896> 2017-11-05 21:59:23.247887 7f23302e4700 20 is_capable service=mon command= read on cap allow *
  -895> 2017-11-05 21:59:23.247890 7f23302e4700 20  allow so far , doing grant allow *
  -894> 2017-11-05 21:59:23.247891 7f23302e4700 20  allow all
  -893> 2017-11-05 21:59:23.247895 7f23302e4700 10 mon.ceph03@2(peon).data_health(344) service_dispatch_op mon_health( service 1 op tell e 0 r 0 ) v1
  -892> 2017-11-05 21:59:23.247900 7f23302e4700 10 mon.ceph03@2(peon).data_health(344) handle_tell mon_health( service 1 op tell e 0 r 0 ) v1
  -891> 2017-11-05 21:59:24.275431 7f232c2dc700 10 _calc_signature seq 248515270 front_crc_ = 1916288138 middle_crc = 0 data_crc = 0 sig = 5259317027298220120
  -890> 2017-11-05 21:59:24.275456 7f232c2dc700  5 -- 10.10.1.23:6789/0 >> 10.10.1.21:6789/0 conn(0x7f2342dd4800 :-1 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=1124301 cs=1 l=0). rx mon.0 seq 248515270 0x7f234296b900 paxos(begin lc 112907750 fc 0 pn 13700 opn 0) v4
  -889> 2017-11-05 21:59:24.275525 7f23302e4700  1 -- 10.10.1.23:6789/0 <== mon.0 10.10.1.21:6789/0 248515270 ==== paxos(begin lc 112907750 fc 0 pn 13700 opn 0) v4 ==== 3352+0+0 (1916288138 0 0) 0x7f234296b900 con 0x7f2342dd4800
  -888> 2017-11-05 21:59:24.275563 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
  -887> 2017-11-05 21:59:24.275574 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
  -886> 2017-11-05 21:59:24.275578 7f23302e4700 20 is_capable service=mon command= read on cap allow *
  -885> 2017-11-05 21:59:24.275583 7f23302e4700 20  allow so far , doing grant allow *
  -884> 2017-11-05 21:59:24.275585 7f23302e4700 20  allow all
  -883> 2017-11-05 21:59:24.275587 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
  -882> 2017-11-05 21:59:24.275589 7f23302e4700 20  allow so far , doing grant allow *
  -881> 2017-11-05 21:59:24.275590 7f23302e4700 20  allow all
  -880> 2017-11-05 21:59:24.275596 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907750) handle_begin paxos(begin lc 112907750 fc 0 pn 13700 opn 0) v4
  -879> 2017-11-05 21:59:24.275602 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos updating c 112907080..112907750) accepting value for 112907751 pn 13700
  -878> 2017-11-05 21:59:24.314323 7f23302e4700  1 -- 10.10.1.23:6789/0 --> 10.10.1.21:6789/0 -- paxos(accept lc 112907750 fc 0 pn 13700 opn 0) v4 -- 0x7f234296b600 con 0
  -877> 2017-11-05 21:59:24.314433 7f232c2dc700 10 _calc_signature seq 36 front_crc_ = 1221148835 middle_crc = 0 data_crc = 0 sig = 13457619710782546457
  -876> 2017-11-05 21:59:24.314453 7f232c2dc700 20 Putting signature in client message(seq # 36): sig = 13457619710782546457
  -875> 2017-11-05 21:59:24.353252 7f232c2dc700 10 _calc_signature seq 248515271 front_crc_ = 4266369837 middle_crc = 0 data_crc = 0 sig = 17507013993340613918
  -874> 2017-11-05 21:59:24.353270 7f232c2dc700  5 -- 10.10.1.23:6789/0 >> 10.10.1.21:6789/0 conn(0x7f2342dd4800 :-1 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=1124301 cs=1 l=0). rx mon.0 seq 248515271 0x7f234296b600 paxos(commit lc 112907751 fc 0 pn 13700 opn 0) v4
  -873> 2017-11-05 21:59:24.353333 7f23302e4700  1 -- 10.10.1.23:6789/0 <== mon.0 10.10.1.21:6789/0 248515271 ==== paxos(commit lc 112907751 fc 0 pn 13700 opn 0) v4 ==== 3352+0+0 (4266369837 0 0) 0x7f234296b600 con 0x7f2342dd4800
  -872> 2017-11-05 21:59:24.353368 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
  -871> 2017-11-05 21:59:24.353377 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
  -870> 2017-11-05 21:59:24.353380 7f23302e4700 20 is_capable service=mon command= read on cap allow *
  -869> 2017-11-05 21:59:24.353383 7f23302e4700 20  allow so far , doing grant allow *
  -868> 2017-11-05 21:59:24.353386 7f23302e4700 20  allow all
  -867> 2017-11-05 21:59:24.353388 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
  -866> 2017-11-05 21:59:24.353390 7f23302e4700 20  allow so far , doing grant allow *
  -865> 2017-11-05 21:59:24.353391 7f23302e4700 20  allow all
  -864> 2017-11-05 21:59:24.353396 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos updating c 112907080..112907750) handle_commit on 112907751
  -863> 2017-11-05 21:59:24.353401 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos updating c 112907080..112907751) store_state [112907751..112907751]
  -862> 2017-11-05 21:59:24.353583 7f232c2dc700 10 _calc_signature seq 248515272 front_crc_ = 1064734685 middle_crc = 0 data_crc = 0 sig = 10177169538778707476
  -861> 2017-11-05 21:59:24.353595 7f232c2dc700  5 -- 10.10.1.23:6789/0 >> 10.10.1.21:6789/0 conn(0x7f2342dd4800 :-1 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=1124301 cs=1 l=0). rx mon.0 seq 248515272 0x7f234296b300 paxos(lease lc 112907751 fc 112907080 pn 0 opn 0) v4
  -860> 2017-11-05 21:59:24.391989 7f23302e4700 10 mon.ceph03@2(peon) e14 refresh_from_paxos
  -859> 2017-11-05 21:59:24.392025 7f23302e4700 10 mon.ceph03@2(peon) e14 refresh_from_paxos no cluster_fingerprint
  -858> 2017-11-05 21:59:24.392060 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(pgmap 56600067..56600783) refresh
  -857> 2017-11-05 21:59:24.392138 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mdsmap 1..1) refresh
  -856> 2017-11-05 21:59:24.392193 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(osdmap 503..1105) refresh
  -855> 2017-11-05 21:59:24.392229 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(logm 55596915..55597637) refresh
  -854> 2017-11-05 21:59:24.392236 7f23302e4700 10 mon.ceph03@2(peon).log v55597637 update_from_paxos
  -853> 2017-11-05 21:59:24.392239 7f23302e4700 10 mon.ceph03@2(peon).log v55597637 update_from_paxos version 55597637 summary v 55597637
  -852> 2017-11-05 21:59:24.392282 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(monmap 1..14) refresh
  -851> 2017-11-05 21:59:24.392309 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(auth 26252..26332) refresh
  -850> 2017-11-05 21:59:24.392315 7f23302e4700 10 mon.ceph03@2(peon).auth v26332 update_from_paxos
  -849> 2017-11-05 21:59:24.392359 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgr 1..394) refresh
  -848> 2017-11-05 21:59:24.392373 7f23302e4700  4 mgrc handle_mgr_map Got map version 394
  -847> 2017-11-05 21:59:24.392376 7f23302e4700  4 mgrc handle_mgr_map Active mgr is now 10.10.1.21:6806/22556
  -846> 2017-11-05 21:59:24.392420 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgrstat 56609513..56609640) refresh
  -845> 2017-11-05 21:59:24.392427 7f23302e4700 10 mon.ceph03@2(peon).mgrstat  56609640
  -844> 2017-11-05 21:59:24.392479 7f23302e4700 10 mon.ceph03@2(peon).mgrstat update_from_paxos v56609640 service_map e1
  -843> 2017-11-05 21:59:24.392485 7f23302e4700 10 mon.ceph03@2(peon).mgrstat check_subs
  -842> 2017-11-05 21:59:24.392488 7f23302e4700 20 mon.ceph03@2(peon).mgrstat update_logger
  -841> 2017-11-05 21:59:24.392523 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(health 1..24) refresh
  -840> 2017-11-05 21:59:24.392530 7f23302e4700 10 mon.ceph03@2(peon).health update_from_paxos
  -839> 2017-11-05 21:59:24.392547 7f23302e4700 20 mon.ceph03@2(peon).health dump:{
    "quorum_health": {},
    "leader_health": {}
}

  -838> 2017-11-05 21:59:24.392563 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(pgmap 56600067..56600783) post_refresh
  -837> 2017-11-05 21:59:24.392565 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mdsmap 1..1) post_refresh
  -836> 2017-11-05 21:59:24.392567 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(osdmap 503..1105) post_refresh
  -835> 2017-11-05 21:59:24.392569 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(logm 55596915..55597637) post_refresh
  -834> 2017-11-05 21:59:24.392571 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(monmap 1..14) post_refresh
  -833> 2017-11-05 21:59:24.392573 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(auth 26252..26332) post_refresh
  -832> 2017-11-05 21:59:24.392575 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgr 1..394) post_refresh
  -831> 2017-11-05 21:59:24.392576 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgrstat 56609513..56609640) post_refresh
  -830> 2017-11-05 21:59:24.392578 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(health 1..24) post_refresh
  -829> 2017-11-05 21:59:24.392664 7f23302e4700  1 -- 10.10.1.23:6789/0 <== mon.0 10.10.1.21:6789/0 248515272 ==== paxos(lease lc 112907751 fc 112907080 pn 0 opn 0) v4 ==== 84+0+0 (1064734685 0 0) 0x7f234296b300 con 0x7f2342dd4800
  -828> 2017-11-05 21:59:24.392682 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
  -827> 2017-11-05 21:59:24.392690 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
  -826> 2017-11-05 21:59:24.392693 7f23302e4700 20 is_capable service=mon command= read on cap allow *
  -825> 2017-11-05 21:59:24.392696 7f23302e4700 20  allow so far , doing grant allow *
  -824> 2017-11-05 21:59:24.392697 7f23302e4700 20  allow all
  -823> 2017-11-05 21:59:24.392698 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
  -822> 2017-11-05 21:59:24.392716 7f23302e4700 20  allow so far , doing grant allow *
  -821> 2017-11-05 21:59:24.392718 7f23302e4700 20  allow all
  -820> 2017-11-05 21:59:24.392722 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907751) handle_lease on 112907751 now 2017-11-05 21:59:29.351713
  -819> 2017-11-05 21:59:24.392734 7f23302e4700  1 -- 10.10.1.23:6789/0 --> 10.10.1.21:6789/0 -- paxos(lease_ack lc 112907751 fc 112907080 pn 0 opn 0) v4 -- 0x7f234296b600 con 0
  -818> 2017-11-05 21:59:24.392755 7f23302e4700 20 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907751) reset_lease_timeout - setting timeout event
  -817> 2017-11-05 21:59:24.392765 7f23302e4700  5 mon.ceph03@2(peon).monmap v14 apply_mon_features wait for service to be writeable
  -816> 2017-11-05 21:59:24.392769 7f23302e4700  5 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907751) is_readable = 1 - now=2017-11-05 21:59:24.392769 lease_expire=2017-11-05 21:59:29.351713 has v0 lc 112907751
  -815> 2017-11-05 21:59:24.392824 7f232c2dc700 10 _calc_signature seq 37 front_crc_ = 2948331090 middle_crc = 0 data_crc = 0 sig = 11223045120982073528
  -814> 2017-11-05 21:59:24.392842 7f232c2dc700 20 Putting signature in client message(seq # 37): sig = 11223045120982073528
  -813> 2017-11-05 21:59:26.268592 7f232c2dc700 10 _calc_signature seq 248515273 front_crc_ = 1773435773 middle_crc = 0 data_crc = 0 sig = 5758794856142174140
  -812> 2017-11-05 21:59:26.268615 7f232c2dc700  5 -- 10.10.1.23:6789/0 >> 10.10.1.21:6789/0 conn(0x7f2342dd4800 :-1 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=1124301 cs=1 l=0). rx mon.0 seq 248515273 0x7f234296b600 paxos(begin lc 112907751 fc 0 pn 13700 opn 0) v4
  -811> 2017-11-05 21:59:26.268696 7f23302e4700  1 -- 10.10.1.23:6789/0 <== mon.0 10.10.1.21:6789/0 248515273 ==== paxos(begin lc 112907751 fc 0 pn 13700 opn 0) v4 ==== 3352+0+0 (1773435773 0 0) 0x7f234296b600 con 0x7f2342dd4800
  -810> 2017-11-05 21:59:26.268737 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
  -809> 2017-11-05 21:59:26.268748 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
  -808> 2017-11-05 21:59:26.268752 7f23302e4700 20 is_capable service=mon command= read on cap allow *
  -807> 2017-11-05 21:59:26.268755 7f23302e4700 20  allow so far , doing grant allow *
  -806> 2017-11-05 21:59:26.268757 7f23302e4700 20  allow all
  -805> 2017-11-05 21:59:26.268759 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
  -804> 2017-11-05 21:59:26.268760 7f23302e4700 20  allow so far , doing grant allow *
  -803> 2017-11-05 21:59:26.268761 7f23302e4700 20  allow all
  -802> 2017-11-05 21:59:26.268768 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907751) handle_begin paxos(begin lc 112907751 fc 0 pn 13700 opn 0) v4
  -801> 2017-11-05 21:59:26.268773 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos updating c 112907080..112907751) accepting value for 112907752 pn 13700
  -800> 2017-11-05 21:59:26.307299 7f23302e4700  1 -- 10.10.1.23:6789/0 --> 10.10.1.21:6789/0 -- paxos(accept lc 112907751 fc 0 pn 13700 opn 0) v4 -- 0x7f234296b300 con 0
  -799> 2017-11-05 21:59:26.307408 7f232c2dc700 10 _calc_signature seq 38 front_crc_ = 382545443 middle_crc = 0 data_crc = 0 sig = 6576343383518060326
  -798> 2017-11-05 21:59:26.307427 7f232c2dc700 20 Putting signature in client message(seq # 38): sig = 6576343383518060326
  -797> 2017-11-05 21:59:26.346943 7f232c2dc700 10 _calc_signature seq 248515274 front_crc_ = 3233349729 middle_crc = 0 data_crc = 0 sig = 4287989692588765716
  -796> 2017-11-05 21:59:26.346960 7f232c2dc700  5 -- 10.10.1.23:6789/0 >> 10.10.1.21:6789/0 conn(0x7f2342dd4800 :-1 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=1124301 cs=1 l=0). rx mon.0 seq 248515274 0x7f234296b300 paxos(commit lc 112907752 fc 0 pn 13700 opn 0) v4
  -795> 2017-11-05 21:59:26.347015 7f232c2dc700 10 _calc_signature seq 248515275 front_crc_ = 3164033032 middle_crc = 0 data_crc = 0 sig = 4655778565917331996
  -794> 2017-11-05 21:59:26.347019 7f232c2dc700  5 -- 10.10.1.23:6789/0 >> 10.10.1.21:6789/0 conn(0x7f2342dd4800 :-1 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=1124301 cs=1 l=0). rx mon.0 seq 248515275 0x7f2342ef2f00 paxos(lease lc 112907752 fc 112907080 pn 0 opn 0) v4
  -793> 2017-11-05 21:59:26.347022 7f23302e4700  1 -- 10.10.1.23:6789/0 <== mon.0 10.10.1.21:6789/0 248515274 ==== paxos(commit lc 112907752 fc 0 pn 13700 opn 0) v4 ==== 3352+0+0 (3233349729 0 0) 0x7f234296b300 con 0x7f2342dd4800
  -792> 2017-11-05 21:59:26.347056 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
  -791> 2017-11-05 21:59:26.347065 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
  -790> 2017-11-05 21:59:26.347068 7f23302e4700 20 is_capable service=mon command= read on cap allow *
  -789> 2017-11-05 21:59:26.347071 7f23302e4700 20  allow so far , doing grant allow *
  -788> 2017-11-05 21:59:26.347072 7f23302e4700 20  allow all
  -787> 2017-11-05 21:59:26.347074 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
  -786> 2017-11-05 21:59:26.347075 7f23302e4700 20  allow so far , doing grant allow *
  -785> 2017-11-05 21:59:26.347076 7f23302e4700 20  allow all
  -784> 2017-11-05 21:59:26.347083 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos updating c 112907080..112907751) handle_commit on 112907752
  -783> 2017-11-05 21:59:26.347088 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos updating c 112907080..112907752) store_state [112907752..112907752]
  -782> 2017-11-05 21:59:26.385029 7f23302e4700 10 mon.ceph03@2(peon) e14 refresh_from_paxos
  -781> 2017-11-05 21:59:26.385064 7f23302e4700 10 mon.ceph03@2(peon) e14 refresh_from_paxos no cluster_fingerprint
  -780> 2017-11-05 21:59:26.385095 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(pgmap 56600067..56600783) refresh
  -779> 2017-11-05 21:59:26.385143 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mdsmap 1..1) refresh
  -778> 2017-11-05 21:59:26.385189 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(osdmap 503..1105) refresh
  -777> 2017-11-05 21:59:26.385223 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(logm 55596915..55597637) refresh
  -776> 2017-11-05 21:59:26.385230 7f23302e4700 10 mon.ceph03@2(peon).log v55597637 update_from_paxos
  -775> 2017-11-05 21:59:26.385233 7f23302e4700 10 mon.ceph03@2(peon).log v55597637 update_from_paxos version 55597637 summary v 55597637
  -774> 2017-11-05 21:59:26.385275 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(monmap 1..14) refresh
  -773> 2017-11-05 21:59:26.385301 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(auth 26252..26332) refresh
  -772> 2017-11-05 21:59:26.385307 7f23302e4700 10 mon.ceph03@2(peon).auth v26332 update_from_paxos
  -771> 2017-11-05 21:59:26.385348 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgr 1..394) refresh
  -770> 2017-11-05 21:59:26.385361 7f23302e4700  4 mgrc handle_mgr_map Got map version 394
  -769> 2017-11-05 21:59:26.385364 7f23302e4700  4 mgrc handle_mgr_map Active mgr is now 10.10.1.21:6806/22556
  -768> 2017-11-05 21:59:26.385405 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgrstat 56609513..56609641) refresh
  -767> 2017-11-05 21:59:26.385412 7f23302e4700 10 mon.ceph03@2(peon).mgrstat  56609641
  -766> 2017-11-05 21:59:26.385461 7f23302e4700 10 mon.ceph03@2(peon).mgrstat update_from_paxos v56609641 service_map e1
  -765> 2017-11-05 21:59:26.385467 7f23302e4700 10 mon.ceph03@2(peon).mgrstat check_subs
  -764> 2017-11-05 21:59:26.385469 7f23302e4700 20 mon.ceph03@2(peon).mgrstat update_logger
  -763> 2017-11-05 21:59:26.385501 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(health 1..24) refresh
  -762> 2017-11-05 21:59:26.385508 7f23302e4700 10 mon.ceph03@2(peon).health update_from_paxos
  -761> 2017-11-05 21:59:26.385524 7f23302e4700 20 mon.ceph03@2(peon).health dump:{
    "quorum_health": {},
    "leader_health": {}
}

  -760> 2017-11-05 21:59:26.385541 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(pgmap 56600067..56600783) post_refresh
  -759> 2017-11-05 21:59:26.385543 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mdsmap 1..1) post_refresh
  -758> 2017-11-05 21:59:26.385545 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(osdmap 503..1105) post_refresh
  -757> 2017-11-05 21:59:26.385546 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(logm 55596915..55597637) post_refresh
  -756> 2017-11-05 21:59:26.385548 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(monmap 1..14) post_refresh
  -755> 2017-11-05 21:59:26.385549 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(auth 26252..26332) post_refresh
  -754> 2017-11-05 21:59:26.385551 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgr 1..394) post_refresh
  -753> 2017-11-05 21:59:26.385552 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgrstat 56609513..56609641) post_refresh
  -752> 2017-11-05 21:59:26.385554 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(health 1..24) post_refresh
  -751> 2017-11-05 21:59:26.385623 7f23302e4700  1 -- 10.10.1.23:6789/0 <== mon.0 10.10.1.21:6789/0 248515275 ==== paxos(lease lc 112907752 fc 112907080 pn 0 opn 0) v4 ==== 84+0+0 (3164033032 0 0) 0x7f2342ef2f00 con 0x7f2342dd4800
  -750> 2017-11-05 21:59:26.385642 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
  -749> 2017-11-05 21:59:26.385650 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
  -748> 2017-11-05 21:59:26.385653 7f23302e4700 20 is_capable service=mon command= read on cap allow *
  -747> 2017-11-05 21:59:26.385656 7f23302e4700 20  allow so far , doing grant allow *
  -746> 2017-11-05 21:59:26.385657 7f23302e4700 20  allow all
  -745> 2017-11-05 21:59:26.385659 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
  -744> 2017-11-05 21:59:26.385660 7f23302e4700 20  allow so far , doing grant allow *
  -743> 2017-11-05 21:59:26.385661 7f23302e4700 20  allow all
  -742> 2017-11-05 21:59:26.385665 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907752) handle_lease on 112907752 now 2017-11-05 21:59:31.344866
  -741> 2017-11-05 21:59:26.385678 7f23302e4700  1 -- 10.10.1.23:6789/0 --> 10.10.1.21:6789/0 -- paxos(lease_ack lc 112907752 fc 112907080 pn 0 opn 0) v4 -- 0x7f234296b300 con 0
  -740> 2017-11-05 21:59:26.385696 7f23302e4700 20 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907752) reset_lease_timeout - setting timeout event
  -739> 2017-11-05 21:59:26.385703 7f23302e4700  5 mon.ceph03@2(peon).monmap v14 apply_mon_features wait for service to be writeable
  -738> 2017-11-05 21:59:26.385706 7f23302e4700  5 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907752) is_readable = 1 - now=2017-11-05 21:59:26.385707 lease_expire=2017-11-05 21:59:31.344866 has v0 lc 112907752
  -737> 2017-11-05 21:59:26.385739 7f232c2dc700 10 _calc_signature seq 39 front_crc_ = 1024609643 middle_crc = 0 data_crc = 0 sig = 15304902292167068210
  -736> 2017-11-05 21:59:26.385750 7f232c2dc700 20 Putting signature in client message(seq # 39): sig = 15304902292167068210
  -735> 2017-11-05 21:59:27.692448 7f2332ae9700 11 mon.ceph03@2(peon) e14 tick
  -734> 2017-11-05 21:59:27.692469 7f2332ae9700 10 mon.ceph03@2(peon).mds e1
  -733> 2017-11-05 21:59:27.692474 7f2332ae9700 10 mon.ceph03@2(peon).osd e1105 e1105: 9 total, 9 up, 9 in
  -732> 2017-11-05 21:59:27.692479 7f2332ae9700 10 mon.ceph03@2(peon).log v55597637 log
  -731> 2017-11-05 21:59:27.692509 7f2332ae9700 10 mon.ceph03@2(peon).auth v26332 auth
  -730> 2017-11-05 21:59:27.692513 7f2332ae9700 10 mon.ceph03@2(peon).health tick
  -729> 2017-11-05 21:59:27.692515 7f2332ae9700 20 mon.ceph03@2(peon).health check_member_health
  -728> 2017-11-05 21:59:27.692599 7f2332ae9700 10 mon.ceph03@2(peon).health check_member_health avail 92% total 277 GB, used 5676 MB, avail 257 GB
  -727> 2017-11-05 21:59:27.692609 7f2332ae9700 20 mon.ceph03@2(peon) e14 sync_trim_providers
  -726> 2017-11-05 21:59:27.898700 7f232f2e2700  1 -- 10.10.1.23:0/15195 --> 10.10.1.21:6806/22556 -- mgrreport(unknown.ceph03 +0-0 packed 1734) v4 -- 0x7f2342beb600 con 0
  -725> 2017-11-05 21:59:27.898811 7f232d2de700 10 _calc_signature seq 5 front_crc_ = 1497291623 middle_crc = 0 data_crc = 0 sig = 7653944384080933906
  -724> 2017-11-05 21:59:27.898830 7f232d2de700 20 Putting signature in client message(seq # 5): sig = 7653944384080933906
  -723> 2017-11-05 21:59:28.273089 7f232c2dc700 10 _calc_signature seq 248515276 front_crc_ = 1793616534 middle_crc = 0 data_crc = 0 sig = 17139277050217223183
  -722> 2017-11-05 21:59:28.273108 7f232c2dc700  5 -- 10.10.1.23:6789/0 >> 10.10.1.21:6789/0 conn(0x7f2342dd4800 :-1 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=1124301 cs=1 l=0). rx mon.0 seq 248515276 0x7f234296b300 paxos(begin lc 112907752 fc 0 pn 13700 opn 0) v4
  -721> 2017-11-05 21:59:28.273166 7f23302e4700  1 -- 10.10.1.23:6789/0 <== mon.0 10.10.1.21:6789/0 248515276 ==== paxos(begin lc 112907752 fc 0 pn 13700 opn 0) v4 ==== 3344+0+0 (1793616534 0 0) 0x7f234296b300 con 0x7f2342dd4800
  -720> 2017-11-05 21:59:28.273202 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
  -719> 2017-11-05 21:59:28.273213 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
  -718> 2017-11-05 21:59:28.273217 7f23302e4700 20 is_capable service=mon command= read on cap allow *
  -717> 2017-11-05 21:59:28.273221 7f23302e4700 20  allow so far , doing grant allow *
  -716> 2017-11-05 21:59:28.273222 7f23302e4700 20  allow all
  -715> 2017-11-05 21:59:28.273225 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
  -714> 2017-11-05 21:59:28.273226 7f23302e4700 20  allow so far , doing grant allow *
  -713> 2017-11-05 21:59:28.273227 7f23302e4700 20  allow all
  -712> 2017-11-05 21:59:28.273235 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907752) handle_begin paxos(begin lc 112907752 fc 0 pn 13700 opn 0) v4
  -711> 2017-11-05 21:59:28.273240 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos updating c 112907080..112907752) accepting value for 112907753 pn 13700
  -710> 2017-11-05 21:59:28.311321 7f23302e4700  1 -- 10.10.1.23:6789/0 --> 10.10.1.21:6789/0 -- paxos(accept lc 112907752 fc 0 pn 13700 opn 0) v4 -- 0x7f2342ef2f00 con 0
  -709> 2017-11-05 21:59:28.311436 7f232c2dc700 10 _calc_signature seq 40 front_crc_ = 2022008981 middle_crc = 0 data_crc = 0 sig = 7818860563918508765
  -708> 2017-11-05 21:59:28.311453 7f232c2dc700 20 Putting signature in client message(seq # 40): sig = 7818860563918508765
  -707> 2017-11-05 21:59:28.339873 7f232c2dc700 10 _calc_signature seq 248515277 front_crc_ = 1315244736 middle_crc = 0 data_crc = 0 sig = 7080838384747451889
  -706> 2017-11-05 21:59:28.339889 7f232c2dc700  5 -- 10.10.1.23:6789/0 >> 10.10.1.21:6789/0 conn(0x7f2342dd4800 :-1 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=1124301 cs=1 l=0). rx mon.0 seq 248515277 0x7f2342ef2f00 paxos(commit lc 112907753 fc 0 pn 13700 opn 0) v4
  -705> 2017-11-05 21:59:28.339957 7f23302e4700  1 -- 10.10.1.23:6789/0 <== mon.0 10.10.1.21:6789/0 248515277 ==== paxos(commit lc 112907753 fc 0 pn 13700 opn 0) v4 ==== 3344+0+0 (1315244736 0 0) 0x7f2342ef2f00 con 0x7f2342dd4800
  -704> 2017-11-05 21:59:28.339990 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
  -703> 2017-11-05 21:59:28.340000 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
  -702> 2017-11-05 21:59:28.340003 7f23302e4700 20 is_capable service=mon command= read on cap allow *
  -701> 2017-11-05 21:59:28.340006 7f23302e4700 20  allow so far , doing grant allow *
  -700> 2017-11-05 21:59:28.340007 7f23302e4700 20  allow all
  -699> 2017-11-05 21:59:28.340009 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
  -698> 2017-11-05 21:59:28.340011 7f23302e4700 20  allow so far , doing grant allow *
  -697> 2017-11-05 21:59:28.340013 7f23302e4700 20  allow all
  -696> 2017-11-05 21:59:28.340038 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos updating c 112907080..112907752) handle_commit on 112907753
  -695> 2017-11-05 21:59:28.340044 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos updating c 112907080..112907753) store_state [112907753..112907753]
  -694> 2017-11-05 21:59:28.340240 7f232c2dc700 10 _calc_signature seq 248515278 front_crc_ = 7690493 middle_crc = 0 data_crc = 0 sig = 12630673617109963947
  -693> 2017-11-05 21:59:28.340252 7f232c2dc700  5 -- 10.10.1.23:6789/0 >> 10.10.1.21:6789/0 conn(0x7f2342dd4800 :-1 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=1124301 cs=1 l=0). rx mon.0 seq 248515278 0x7f2342ef2c00 paxos(lease lc 112907753 fc 112907080 pn 0 opn 0) v4
  -692> 2017-11-05 21:59:28.378021 7f23302e4700 10 mon.ceph03@2(peon) e14 refresh_from_paxos
  -691> 2017-11-05 21:59:28.378058 7f23302e4700 10 mon.ceph03@2(peon) e14 refresh_from_paxos no cluster_fingerprint
  -690> 2017-11-05 21:59:28.378091 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(pgmap 56600067..56600783) refresh
  -689> 2017-11-05 21:59:28.378141 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mdsmap 1..1) refresh
  -688> 2017-11-05 21:59:28.378191 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(osdmap 503..1105) refresh
  -687> 2017-11-05 21:59:28.378227 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(logm 55596915..55597637) refresh
  -686> 2017-11-05 21:59:28.378234 7f23302e4700 10 mon.ceph03@2(peon).log v55597637 update_from_paxos
  -685> 2017-11-05 21:59:28.378238 7f23302e4700 10 mon.ceph03@2(peon).log v55597637 update_from_paxos version 55597637 summary v 55597637
  -684> 2017-11-05 21:59:28.378280 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(monmap 1..14) refresh
  -683> 2017-11-05 21:59:28.378308 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(auth 26252..26332) refresh
  -682> 2017-11-05 21:59:28.378315 7f23302e4700 10 mon.ceph03@2(peon).auth v26332 update_from_paxos
  -681> 2017-11-05 21:59:28.378359 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgr 1..394) refresh
  -680> 2017-11-05 21:59:28.378373 7f23302e4700  4 mgrc handle_mgr_map Got map version 394
  -679> 2017-11-05 21:59:28.378377 7f23302e4700  4 mgrc handle_mgr_map Active mgr is now 10.10.1.21:6806/22556
  -678> 2017-11-05 21:59:28.378414 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgrstat 56609513..56609642) refresh
  -677> 2017-11-05 21:59:28.378421 7f23302e4700 10 mon.ceph03@2(peon).mgrstat  56609642
  -676> 2017-11-05 21:59:28.378472 7f23302e4700 10 mon.ceph03@2(peon).mgrstat update_from_paxos v56609642 service_map e1
  -675> 2017-11-05 21:59:28.378479 7f23302e4700 10 mon.ceph03@2(peon).mgrstat check_subs
  -674> 2017-11-05 21:59:28.378481 7f23302e4700 20 mon.ceph03@2(peon).mgrstat update_logger
  -673> 2017-11-05 21:59:28.378514 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(health 1..24) refresh
  -672> 2017-11-05 21:59:28.378521 7f23302e4700 10 mon.ceph03@2(peon).health update_from_paxos
  -671> 2017-11-05 21:59:28.378539 7f23302e4700 20 mon.ceph03@2(peon).health dump:{
    "quorum_health": {},
    "leader_health": {}
}

  -670> 2017-11-05 21:59:28.378555 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(pgmap 56600067..56600783) post_refresh
  -669> 2017-11-05 21:59:28.378557 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mdsmap 1..1) post_refresh
  -668> 2017-11-05 21:59:28.378559 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(osdmap 503..1105) post_refresh
  -667> 2017-11-05 21:59:28.378561 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(logm 55596915..55597637) post_refresh
  -666> 2017-11-05 21:59:28.378562 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(monmap 1..14) post_refresh
  -665> 2017-11-05 21:59:28.378564 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(auth 26252..26332) post_refresh
  -664> 2017-11-05 21:59:28.378566 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgr 1..394) post_refresh
  -663> 2017-11-05 21:59:28.378567 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgrstat 56609513..56609642) post_refresh
  -662> 2017-11-05 21:59:28.378570 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(health 1..24) post_refresh
  -661> 2017-11-05 21:59:28.378641 7f23302e4700  1 -- 10.10.1.23:6789/0 <== mon.0 10.10.1.21:6789/0 248515278 ==== paxos(lease lc 112907753 fc 112907080 pn 0 opn 0) v4 ==== 84+0+0 (7690493 0 0) 0x7f2342ef2c00 con 0x7f2342dd4800
  -660> 2017-11-05 21:59:28.378661 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
  -659> 2017-11-05 21:59:28.378666 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
  -658> 2017-11-05 21:59:28.378669 7f23302e4700 20 is_capable service=mon command= read on cap allow *
  -657> 2017-11-05 21:59:28.378674 7f23302e4700 20  allow so far , doing grant allow *
  -656> 2017-11-05 21:59:28.378675 7f23302e4700 20  allow all
  -655> 2017-11-05 21:59:28.378677 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
  -654> 2017-11-05 21:59:28.378678 7f23302e4700 20  allow so far , doing grant allow *
  -653> 2017-11-05 21:59:28.378679 7f23302e4700 20  allow all
  -652> 2017-11-05 21:59:28.378684 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907753) handle_lease on 112907753 now 2017-11-05 21:59:33.338364
  -651> 2017-11-05 21:59:28.378698 7f23302e4700  1 -- 10.10.1.23:6789/0 --> 10.10.1.21:6789/0 -- paxos(lease_ack lc 112907753 fc 112907080 pn 0 opn 0) v4 -- 0x7f2342ef2f00 con 0
  -650> 2017-11-05 21:59:28.378716 7f23302e4700 20 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907753) reset_lease_timeout - setting timeout event
  -649> 2017-11-05 21:59:28.378723 7f23302e4700  5 mon.ceph03@2(peon).monmap v14 apply_mon_features wait for service to be writeable
  -648> 2017-11-05 21:59:28.378726 7f23302e4700  5 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907753) is_readable = 1 - now=2017-11-05 21:59:28.378727 lease_expire=2017-11-05 21:59:33.338364 has v0 lc 112907753
  -647> 2017-11-05 21:59:28.378784 7f232c2dc700 10 _calc_signature seq 41 front_crc_ = 4223284809 middle_crc = 0 data_crc = 0 sig = 2790373739108272567
  -646> 2017-11-05 21:59:28.378801 7f232c2dc700 20 Putting signature in client message(seq # 41): sig = 2790373739108272567
  -645> 2017-11-05 21:59:30.266286 7f232c2dc700 10 _calc_signature seq 248515279 front_crc_ = 4264070926 middle_crc = 0 data_crc = 0 sig = 17570471818085489431
  -644> 2017-11-05 21:59:30.266318 7f232c2dc700  5 -- 10.10.1.23:6789/0 >> 10.10.1.21:6789/0 conn(0x7f2342dd4800 :-1 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=1124301 cs=1 l=0). rx mon.0 seq 248515279 0x7f2342ef2f00 paxos(begin lc 112907753 fc 0 pn 13700 opn 0) v4
  -643> 2017-11-05 21:59:30.266377 7f23302e4700  1 -- 10.10.1.23:6789/0 <== mon.0 10.10.1.21:6789/0 248515279 ==== paxos(begin lc 112907753 fc 0 pn 13700 opn 0) v4 ==== 3336+0+0 (4264070926 0 0) 0x7f2342ef2f00 con 0x7f2342dd4800
  -642> 2017-11-05 21:59:30.266414 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
  -641> 2017-11-05 21:59:30.266424 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
  -640> 2017-11-05 21:59:30.266429 7f23302e4700 20 is_capable service=mon command= read on cap allow *
  -639> 2017-11-05 21:59:30.266433 7f23302e4700 20  allow so far , doing grant allow *
  -638> 2017-11-05 21:59:30.266434 7f23302e4700 20  allow all
  -637> 2017-11-05 21:59:30.266436 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
  -636> 2017-11-05 21:59:30.266438 7f23302e4700 20  allow so far , doing grant allow *
  -635> 2017-11-05 21:59:30.266439 7f23302e4700 20  allow all
  -634> 2017-11-05 21:59:30.266447 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907753) handle_begin paxos(begin lc 112907753 fc 0 pn 13700 opn 0) v4
  -633> 2017-11-05 21:59:30.266453 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos updating c 112907080..112907753) accepting value for 112907754 pn 13700
  -632> 2017-11-05 21:59:30.304280 7f23302e4700  1 -- 10.10.1.23:6789/0 --> 10.10.1.21:6789/0 -- paxos(accept lc 112907753 fc 0 pn 13700 opn 0) v4 -- 0x7f2342ef2c00 con 0
  -631> 2017-11-05 21:59:30.304357 7f232c2dc700 10 _calc_signature seq 42 front_crc_ = 3454116069 middle_crc = 0 data_crc = 0 sig = 17816842680307483109
  -630> 2017-11-05 21:59:30.304370 7f232c2dc700 20 Putting signature in client message(seq # 42): sig = 17816842680307483109
  -629> 2017-11-05 21:59:30.344058 7f232c2dc700 10 _calc_signature seq 248515280 front_crc_ = 2165871158 middle_crc = 0 data_crc = 0 sig = 17177650932287336935
  -628> 2017-11-05 21:59:30.344074 7f232c2dc700  5 -- 10.10.1.23:6789/0 >> 10.10.1.21:6789/0 conn(0x7f2342dd4800 :-1 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=1124301 cs=1 l=0). rx mon.0 seq 248515280 0x7f2342ef2c00 paxos(commit lc 112907754 fc 0 pn 13700 opn 0) v4
  -627> 2017-11-05 21:59:30.344127 7f23302e4700  1 -- 10.10.1.23:6789/0 <== mon.0 10.10.1.21:6789/0 248515280 ==== paxos(commit lc 112907754 fc 0 pn 13700 opn 0) v4 ==== 3336+0+0 (2165871158 0 0) 0x7f2342ef2c00 con 0x7f2342dd4800
  -626> 2017-11-05 21:59:30.344156 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
  -625> 2017-11-05 21:59:30.344168 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
  -624> 2017-11-05 21:59:30.344171 7f23302e4700 20 is_capable service=mon command= read on cap allow *
  -623> 2017-11-05 21:59:30.344174 7f23302e4700 20  allow so far , doing grant allow *
  -622> 2017-11-05 21:59:30.344195 7f23302e4700 20  allow all
  -621> 2017-11-05 21:59:30.344197 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
  -620> 2017-11-05 21:59:30.344199 7f23302e4700 20  allow so far , doing grant allow *
  -619> 2017-11-05 21:59:30.344200 7f23302e4700 20  allow all
  -618> 2017-11-05 21:59:30.344211 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos updating c 112907080..112907753) handle_commit on 112907754
  -617> 2017-11-05 21:59:30.344217 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos updating c 112907080..112907754) store_state [112907754..112907754]
  -616> 2017-11-05 21:59:30.344428 7f232c2dc700 10 _calc_signature seq 248515281 front_crc_ = 1698768348 middle_crc = 0 data_crc = 0 sig = 16000968398340812496
  -615> 2017-11-05 21:59:30.344440 7f232c2dc700  5 -- 10.10.1.23:6789/0 >> 10.10.1.21:6789/0 conn(0x7f2342dd4800 :-1 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=1124301 cs=1 l=0). rx mon.0 seq 248515281 0x7f2342ef2900 paxos(lease lc 112907754 fc 112907080 pn 0 opn 0) v4
  -614> 2017-11-05 21:59:30.382036 7f23302e4700 10 mon.ceph03@2(peon) e14 refresh_from_paxos
  -613> 2017-11-05 21:59:30.382072 7f23302e4700 10 mon.ceph03@2(peon) e14 refresh_from_paxos no cluster_fingerprint
  -612> 2017-11-05 21:59:30.382106 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(pgmap 56600067..56600783) refresh
  -611> 2017-11-05 21:59:30.382153 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mdsmap 1..1) refresh
  -610> 2017-11-05 21:59:30.382204 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(osdmap 503..1105) refresh
  -609> 2017-11-05 21:59:30.382251 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(logm 55596915..55597637) refresh
  -608> 2017-11-05 21:59:30.382258 7f23302e4700 10 mon.ceph03@2(peon).log v55597637 update_from_paxos
  -607> 2017-11-05 21:59:30.382261 7f23302e4700 10 mon.ceph03@2(peon).log v55597637 update_from_paxos version 55597637 summary v 55597637
  -606> 2017-11-05 21:59:30.382301 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(monmap 1..14) refresh
  -605> 2017-11-05 21:59:30.382326 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(auth 26252..26332) refresh
  -604> 2017-11-05 21:59:30.382333 7f23302e4700 10 mon.ceph03@2(peon).auth v26332 update_from_paxos
  -603> 2017-11-05 21:59:30.382374 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgr 1..394) refresh
  -602> 2017-11-05 21:59:30.382388 7f23302e4700  4 mgrc handle_mgr_map Got map version 394
  -601> 2017-11-05 21:59:30.382391 7f23302e4700  4 mgrc handle_mgr_map Active mgr is now 10.10.1.21:6806/22556
  -600> 2017-11-05 21:59:30.382426 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgrstat 56609513..56609643) refresh
  -599> 2017-11-05 21:59:30.382434 7f23302e4700 10 mon.ceph03@2(peon).mgrstat  56609643
  -598> 2017-11-05 21:59:30.382491 7f23302e4700 10 mon.ceph03@2(peon).mgrstat update_from_paxos v56609643 service_map e1
  -597> 2017-11-05 21:59:30.382497 7f23302e4700 10 mon.ceph03@2(peon).mgrstat check_subs
  -596> 2017-11-05 21:59:30.382502 7f23302e4700 20 mon.ceph03@2(peon).mgrstat update_logger
  -595> 2017-11-05 21:59:30.382534 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(health 1..24) refresh
  -594> 2017-11-05 21:59:30.382541 7f23302e4700 10 mon.ceph03@2(peon).health update_from_paxos
  -593> 2017-11-05 21:59:30.382558 7f23302e4700 20 mon.ceph03@2(peon).health dump:{
    "quorum_health": {},
    "leader_health": {}
}

  -592> 2017-11-05 21:59:30.382573 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(pgmap 56600067..56600783) post_refresh
  -591> 2017-11-05 21:59:30.382575 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mdsmap 1..1) post_refresh
  -590> 2017-11-05 21:59:30.382577 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(osdmap 503..1105) post_refresh
  -589> 2017-11-05 21:59:30.382579 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(logm 55596915..55597637) post_refresh
  -588> 2017-11-05 21:59:30.382580 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(monmap 1..14) post_refresh
  -587> 2017-11-05 21:59:30.382582 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(auth 26252..26332) post_refresh
  -586> 2017-11-05 21:59:30.382583 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgr 1..394) post_refresh
  -585> 2017-11-05 21:59:30.382585 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgrstat 56609513..56609643) post_refresh
  -584> 2017-11-05 21:59:30.382587 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(health 1..24) post_refresh
  -583> 2017-11-05 21:59:30.382659 7f23302e4700  1 -- 10.10.1.23:6789/0 <== mon.0 10.10.1.21:6789/0 248515281 ==== paxos(lease lc 112907754 fc 112907080 pn 0 opn 0) v4 ==== 84+0+0 (1698768348 0 0) 0x7f2342ef2900 con 0x7f2342dd4800
  -582> 2017-11-05 21:59:30.382678 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
  -581> 2017-11-05 21:59:30.382685 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
  -580> 2017-11-05 21:59:30.382688 7f23302e4700 20 is_capable service=mon command= read on cap allow *
  -579> 2017-11-05 21:59:30.382690 7f23302e4700 20  allow so far , doing grant allow *
  -578> 2017-11-05 21:59:30.382691 7f23302e4700 20  allow all
  -577> 2017-11-05 21:59:30.382693 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
  -576> 2017-11-05 21:59:30.382694 7f23302e4700 20  allow so far , doing grant allow *
  -575> 2017-11-05 21:59:30.382696 7f23302e4700 20  allow all
  -574> 2017-11-05 21:59:30.382700 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907754) handle_lease on 112907754 now 2017-11-05 21:59:35.342590
  -573> 2017-11-05 21:59:30.382715 7f23302e4700  1 -- 10.10.1.23:6789/0 --> 10.10.1.21:6789/0 -- paxos(lease_ack lc 112907754 fc 112907080 pn 0 opn 0) v4 -- 0x7f2342ef2c00 con 0
  -572> 2017-11-05 21:59:30.382733 7f23302e4700 20 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907754) reset_lease_timeout - setting timeout event
  -571> 2017-11-05 21:59:30.382742 7f23302e4700  5 mon.ceph03@2(peon).monmap v14 apply_mon_features wait for service to be writeable
  -570> 2017-11-05 21:59:30.382745 7f23302e4700  5 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907754) is_readable = 1 - now=2017-11-05 21:59:30.382746 lease_expire=2017-11-05 21:59:35.342590 has v0 lc 112907754
  -569> 2017-11-05 21:59:30.382781 7f232c2dc700 10 _calc_signature seq 43 front_crc_ = 2373860913 middle_crc = 0 data_crc = 0 sig = 14432907233122927278
  -568> 2017-11-05 21:59:30.382797 7f232c2dc700 20 Putting signature in client message(seq # 43): sig = 14432907233122927278
  -567> 2017-11-05 21:59:30.724091 7f232d2de700  1 -- 10.10.1.23:6789/0 >> - conn(0x7f2342f09000 :6789 s=STATE_ACCEPTING pgs=0 cs=0 l=0)._process_connection sd=37 -
  -566> 2017-11-05 21:59:30.724292 7f232d2de700 10 mon.ceph03@2(peon) e14 ms_verify_authorizer 10.10.1.21:0/1168097192 client protocol 0
  -565> 2017-11-05 21:59:30.724316 7f232d2de700 10 In get_auth_session_handler for protocol 0
  -564> 2017-11-05 21:59:30.724356 7f232d2de700  2 -- 10.10.1.23:6789/0 >> 10.10.1.21:0/1168097192 conn(0x7f2342f09000 :6789 s=STATE_ACCEPTING_WAIT_SEQ pgs=1 cs=1 l=1).handle_connect_msg accept write reply msg done
  -563> 2017-11-05 21:59:30.724486 7f232d2de700  2 -- 10.10.1.23:6789/0 >> 10.10.1.21:0/1168097192 conn(0x7f2342f09000 :6789 s=STATE_ACCEPTING_WAIT_SEQ pgs=1 cs=1 l=1)._process_connection accept get newly_acked_seq 0
  -562> 2017-11-05 21:59:30.724561 7f232d2de700  5 -- 10.10.1.23:6789/0 >> 10.10.1.21:0/1168097192 conn(0x7f2342f09000 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=1 cs=1 l=1). rx client.? seq 1 0x7f2342f14000 auth(proto 0 30 bytes epoch 0) v1
  -561> 2017-11-05 21:59:30.724609 7f23302e4700  1 -- 10.10.1.23:6789/0 <== client.? 10.10.1.21:0/1168097192 1 ==== auth(proto 0 30 bytes epoch 0) v1 ==== 60+0+0 (673663173 0 0) 0x7f2342f14000 con 0x7f2342f09000
  -560> 2017-11-05 21:59:30.724638 7f23302e4700 10 mon.ceph03@2(peon) e14 _ms_dispatch new session 0x7f2342933600 MonSession(client.? 10.10.1.21:0/1168097192 is open ) features 0x1ffddff8eea4fffb
  -559> 2017-11-05 21:59:30.724650 7f23302e4700 20 mon.ceph03@2(peon) e14  caps
  -558> 2017-11-05 21:59:30.724653 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(auth 26252..26332) dispatch 0x7f2342f14000 auth(proto 0 30 bytes epoch 0) v1 from client.? 10.10.1.21:0/1168097192 con 0x7f2342f09000
  -557> 2017-11-05 21:59:30.724661 7f23302e4700  5 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907754) is_readable = 1 - now=2017-11-05 21:59:30.724661 lease_expire=2017-11-05 21:59:35.342590 has v0 lc 112907754
  -556> 2017-11-05 21:59:30.724671 7f23302e4700 10 mon.ceph03@2(peon).auth v26332 preprocess_query auth(proto 0 30 bytes epoch 0) v1 from client.? 10.10.1.21:0/1168097192
  -555> 2017-11-05 21:59:30.724676 7f23302e4700 10 mon.ceph03@2(peon).auth v26332 prep_auth() blob_size=30
  -554> 2017-11-05 21:59:30.724691 7f23302e4700 10 mon.ceph03@2(peon).auth v26332 AuthMonitor::assign_global_id m=auth(proto 0 30 bytes epoch 0) v1 mon=2/3 last_allocated=55234096 max_global_id=55244096
  -553> 2017-11-05 21:59:30.724695 7f23302e4700 10 mon.ceph03@2(peon).auth v26332 next_global_id should be 55234100
  -552> 2017-11-05 21:59:30.724713 7f23302e4700  1 -- 10.10.1.23:6789/0 --> 10.10.1.21:0/1168097192 -- mon_map magic: 0 v1 -- 0x7f234298fcc0 con 0
  -551> 2017-11-05 21:59:30.724755 7f23302e4700 10 cephx server client.admin: start_session server_challenge d3423fceec82c2fe
  -550> 2017-11-05 21:59:30.724764 7f23302e4700  2 mon.ceph03@2(peon) e14 send_reply 0x7f2342e9b8c0 0x7f234296d900 auth_reply(proto 2 0 (0) Success) v1
  -549> 2017-11-05 21:59:30.724777 7f23302e4700  1 -- 10.10.1.23:6789/0 --> 10.10.1.21:0/1168097192 -- auth_reply(proto 2 0 (0) Success) v1 -- 0x7f234296d900 con 0
  -548> 2017-11-05 21:59:30.725066 7f232d2de700  5 -- 10.10.1.23:6789/0 >> 10.10.1.21:0/1168097192 conn(0x7f2342f09000 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=1 cs=1 l=1). rx client.? seq 2 0x7f234296d900 auth(proto 2 32 bytes epoch 0) v1
  -547> 2017-11-05 21:59:30.725147 7f23302e4700  1 -- 10.10.1.23:6789/0 <== client.? 10.10.1.21:0/1168097192 2 ==== auth(proto 2 32 bytes epoch 0) v1 ==== 62+0+0 (621536380 0 0) 0x7f234296d900 con 0x7f2342f09000
  -546> 2017-11-05 21:59:30.725169 7f232d2de700  1 -- 10.10.1.23:6789/0 >> 10.10.1.21:0/1168097192 conn(0x7f2342f09000 :6789 s=STATE_OPEN pgs=1 cs=1 l=1).read_bulk peer close file descriptor 37
  -545> 2017-11-05 21:59:30.725179 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342933600 for client.? 10.10.1.21:0/1168097192
  -544> 2017-11-05 21:59:30.725185 7f23302e4700 20 mon.ceph03@2(peon) e14  caps
  -543> 2017-11-05 21:59:30.725184 7f232d2de700  1 -- 10.10.1.23:6789/0 >> 10.10.1.21:0/1168097192 conn(0x7f2342f09000 :6789 s=STATE_OPEN pgs=1 cs=1 l=1).read_until read failed
  -542> 2017-11-05 21:59:30.725188 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(auth 26252..26332) dispatch 0x7f234296d900 auth(proto 2 32 bytes epoch 0) v1 from client.? 10.10.1.21:0/1168097192 con 0x7f2342f09000
  -541> 2017-11-05 21:59:30.725190 7f232d2de700  1 -- 10.10.1.23:6789/0 >> 10.10.1.21:0/1168097192 conn(0x7f2342f09000 :6789 s=STATE_OPEN pgs=1 cs=1 l=1).process read tag failed
  -540> 2017-11-05 21:59:30.725195 7f232d2de700  1 -- 10.10.1.23:6789/0 >> 10.10.1.21:0/1168097192 conn(0x7f2342f09000 :6789 s=STATE_OPEN pgs=1 cs=1 l=1).fault on lossy channel, failing
  -539> 2017-11-05 21:59:30.725195 7f23302e4700  5 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907754) is_readable = 1 - now=2017-11-05 21:59:30.725195 lease_expire=2017-11-05 21:59:35.342590 has v0 lc 112907754
  -538> 2017-11-05 21:59:30.725200 7f232d2de700  2 -- 10.10.1.23:6789/0 >> 10.10.1.21:0/1168097192 conn(0x7f2342f09000 :6789 s=STATE_OPEN pgs=1 cs=1 l=1)._stop
  -537> 2017-11-05 21:59:30.725204 7f23302e4700 10 mon.ceph03@2(peon).auth v26332 preprocess_query auth(proto 2 32 bytes epoch 0) v1 from client.? 10.10.1.21:0/1168097192
  -536> 2017-11-05 21:59:30.725210 7f23302e4700 10 mon.ceph03@2(peon).auth v26332 prep_auth() blob_size=32
  -535> 2017-11-05 21:59:30.725213 7f23302e4700 10 cephx server client.admin: handle_request get_auth_session_key for client.admin
  -534> 2017-11-05 21:59:30.725247 7f23302e4700 20 cephx server client.admin:  checking key: req.key=e3c606c55c103797 expected_key=e3c606c55c103797
  -533> 2017-11-05 21:59:30.725324 7f23302e4700 10 cephx: build_service_ticket_reply encoding 1 tickets with secret AQAh1Z5VjAN3ARAAw+R6HvdEfcFucLX1YSmG4Q==
  -532> 2017-11-05 21:59:30.725352 7f23302e4700 10 cephx: build_service_ticket service auth secret_id 1699 ticket_info.ticket.name=client.admin
  -531> 2017-11-05 21:59:30.725372 7f23302e4700 10 cephx keyserverdata: get_caps: name=client.admin
  -530> 2017-11-05 21:59:30.725378 7f23302e4700 10 cephx keyserverdata: get_secret: num of caps=4
  -529> 2017-11-05 21:59:30.725432 7f23302e4700  2 mon.ceph03@2(peon) e14 send_reply 0x7f2342e9b8c0 0x7f2342f14000 auth_reply(proto 2 0 (0) Success) v1
  -528> 2017-11-05 21:59:30.725450 7f23302e4700  1 -- 10.10.1.23:6789/0 --> 10.10.1.21:0/1168097192 -- auth_reply(proto 2 0 (0) Success) v1 -- 0x7f2342f14000 con 0
  -527> 2017-11-05 21:59:30.725481 7f23302e4700 10 mon.ceph03@2(peon) e14 ms_handle_reset 0x7f2342f09000 10.10.1.21:0/1168097192
  -526> 2017-11-05 21:59:30.725489 7f23302e4700 10 mon.ceph03@2(peon) e14 reset/close on session client.? 10.10.1.21:0/1168097192
  -525> 2017-11-05 21:59:30.725494 7f23302e4700 10 mon.ceph03@2(peon) e14 remove_session 0x7f2342933600 client.? 10.10.1.21:0/1168097192 features 0x1ffddff8eea4fffb
  -524> 2017-11-05 21:59:30.772133 7f232d2de700  1 -- 10.10.1.23:6789/0 >> - conn(0x7f2342f07800 :6789 s=STATE_ACCEPTING pgs=0 cs=0 l=0)._process_connection sd=37 -
  -523> 2017-11-05 21:59:30.772416 7f232d2de700 10 mon.ceph03@2(peon) e14 ms_verify_authorizer 10.10.1.21:0/1317201829 client protocol 0
  -522> 2017-11-05 21:59:30.772438 7f232d2de700 10 In get_auth_session_handler for protocol 0
  -521> 2017-11-05 21:59:30.772464 7f232d2de700  2 -- 10.10.1.23:6789/0 >> 10.10.1.21:0/1317201829 conn(0x7f2342f07800 :6789 s=STATE_ACCEPTING_WAIT_SEQ pgs=2 cs=1 l=1).handle_connect_msg accept write reply msg done
  -520> 2017-11-05 21:59:30.772539 7f232d2de700  2 -- 10.10.1.23:6789/0 >> 10.10.1.21:0/1317201829 conn(0x7f2342f07800 :6789 s=STATE_ACCEPTING_WAIT_SEQ pgs=2 cs=1 l=1)._process_connection accept get newly_acked_seq 0
  -519> 2017-11-05 21:59:30.772600 7f232d2de700  5 -- 10.10.1.23:6789/0 >> 10.10.1.21:0/1317201829 conn(0x7f2342f07800 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=2 cs=1 l=1). rx client.? seq 1 0x7f2342f14500 auth(proto 0 30 bytes epoch 0) v1
  -518> 2017-11-05 21:59:30.772637 7f23302e4700  1 -- 10.10.1.23:6789/0 <== client.? 10.10.1.21:0/1317201829 1 ==== auth(proto 0 30 bytes epoch 0) v1 ==== 60+0+0 (673663173 0 0) 0x7f2342f14500 con 0x7f2342f07800
  -517> 2017-11-05 21:59:30.772666 7f23302e4700 10 mon.ceph03@2(peon) e14 _ms_dispatch new session 0x7f2342933600 MonSession(client.? 10.10.1.21:0/1317201829 is open ) features 0x1ffddff8eea4fffb
  -516> 2017-11-05 21:59:30.772677 7f23302e4700 20 mon.ceph03@2(peon) e14  caps
  -515> 2017-11-05 21:59:30.772680 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(auth 26252..26332) dispatch 0x7f2342f14500 auth(proto 0 30 bytes epoch 0) v1 from client.? 10.10.1.21:0/1317201829 con 0x7f2342f07800
  -514> 2017-11-05 21:59:30.772686 7f23302e4700  5 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907754) is_readable = 1 - now=2017-11-05 21:59:30.772687 lease_expire=2017-11-05 21:59:35.342590 has v0 lc 112907754
  -513> 2017-11-05 21:59:30.772695 7f23302e4700 10 mon.ceph03@2(peon).auth v26332 preprocess_query auth(proto 0 30 bytes epoch 0) v1 from client.? 10.10.1.21:0/1317201829
  -512> 2017-11-05 21:59:30.772700 7f23302e4700 10 mon.ceph03@2(peon).auth v26332 prep_auth() blob_size=30
  -511> 2017-11-05 21:59:30.772709 7f23302e4700 10 mon.ceph03@2(peon).auth v26332 AuthMonitor::assign_global_id m=auth(proto 0 30 bytes epoch 0) v1 mon=2/3 last_allocated=55234100 max_global_id=55244096
  -510> 2017-11-05 21:59:30.772714 7f23302e4700 10 mon.ceph03@2(peon).auth v26332 next_global_id should be 55234103
  -509> 2017-11-05 21:59:30.772732 7f23302e4700  1 -- 10.10.1.23:6789/0 --> 10.10.1.21:0/1317201829 -- mon_map magic: 0 v1 -- 0x7f234298fa80 con 0
  -508> 2017-11-05 21:59:30.772763 7f23302e4700 10 cephx server client.admin: start_session server_challenge 5c696d6cbed94d3c
  -507> 2017-11-05 21:59:30.772771 7f23302e4700  2 mon.ceph03@2(peon) e14 send_reply 0x7f2342e9d4e0 0x7f234296d900 auth_reply(proto 2 0 (0) Success) v1
  -506> 2017-11-05 21:59:30.772782 7f23302e4700  1 -- 10.10.1.23:6789/0 --> 10.10.1.21:0/1317201829 -- auth_reply(proto 2 0 (0) Success) v1 -- 0x7f234296d900 con 0
  -505> 2017-11-05 21:59:30.773164 7f232d2de700  5 -- 10.10.1.23:6789/0 >> 10.10.1.21:0/1317201829 conn(0x7f2342f07800 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=2 cs=1 l=1). rx client.? seq 2 0x7f234296d900 auth(proto 2 32 bytes epoch 0) v1
  -504> 2017-11-05 21:59:30.773211 7f23302e4700  1 -- 10.10.1.23:6789/0 <== client.? 10.10.1.21:0/1317201829 2 ==== auth(proto 2 32 bytes epoch 0) v1 ==== 62+0+0 (2070786932 0 0) 0x7f234296d900 con 0x7f2342f07800
  -503> 2017-11-05 21:59:30.773238 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342933600 for client.? 10.10.1.21:0/1317201829
  -502> 2017-11-05 21:59:30.773247 7f23302e4700 20 mon.ceph03@2(peon) e14  caps
  -501> 2017-11-05 21:59:30.773250 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(auth 26252..26332) dispatch 0x7f234296d900 auth(proto 2 32 bytes epoch 0) v1 from client.? 10.10.1.21:0/1317201829 con 0x7f2342f07800
  -500> 2017-11-05 21:59:30.773256 7f23302e4700  5 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907754) is_readable = 1 - now=2017-11-05 21:59:30.773257 lease_expire=2017-11-05 21:59:35.342590 has v0 lc 112907754
  -499> 2017-11-05 21:59:30.773266 7f23302e4700 10 mon.ceph03@2(peon).auth v26332 preprocess_query auth(proto 2 32 bytes epoch 0) v1 from client.? 10.10.1.21:0/1317201829
  -498> 2017-11-05 21:59:30.773271 7f23302e4700 10 mon.ceph03@2(peon).auth v26332 prep_auth() blob_size=32
  -497> 2017-11-05 21:59:30.773274 7f23302e4700 10 cephx server client.admin: handle_request get_auth_session_key for client.admin
  -496> 2017-11-05 21:59:30.773302 7f23302e4700 20 cephx server client.admin:  checking key: req.key=371798402e1722a2 expected_key=371798402e1722a2
  -495> 2017-11-05 21:59:30.773355 7f23302e4700 10 cephx: build_service_ticket_reply encoding 1 tickets with secret AQAh1Z5VjAN3ARAAw+R6HvdEfcFucLX1YSmG4Q==
  -494> 2017-11-05 21:59:30.773379 7f23302e4700 10 cephx: build_service_ticket service auth secret_id 1699 ticket_info.ticket.name=client.admin
  -493> 2017-11-05 21:59:30.773394 7f23302e4700 10 cephx keyserverdata: get_caps: name=client.admin
  -492> 2017-11-05 21:59:30.773399 7f23302e4700 10 cephx keyserverdata: get_secret: num of caps=4
  -491> 2017-11-05 21:59:30.773447 7f23302e4700  2 mon.ceph03@2(peon) e14 send_reply 0x7f2342e9d4e0 0x7f2342f14500 auth_reply(proto 2 0 (0) Success) v1
  -490> 2017-11-05 21:59:30.773462 7f23302e4700  1 -- 10.10.1.23:6789/0 --> 10.10.1.21:0/1317201829 -- auth_reply(proto 2 0 (0) Success) v1 -- 0x7f2342f14500 con 0
  -489> 2017-11-05 21:59:30.773856 7f232d2de700  5 -- 10.10.1.23:6789/0 >> 10.10.1.21:0/1317201829 conn(0x7f2342f07800 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=2 cs=1 l=1). rx client.? seq 3 0x7f2342f14500 auth(proto 2 165 bytes epoch 0) v1
  -488> 2017-11-05 21:59:30.773924 7f23302e4700  1 -- 10.10.1.23:6789/0 <== client.? 10.10.1.21:0/1317201829 3 ==== auth(proto 2 165 bytes epoch 0) v1 ==== 195+0+0 (725147063 0 0) 0x7f2342f14500 con 0x7f2342f07800
  -487> 2017-11-05 21:59:30.773958 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342933600 for client.? 10.10.1.21:0/1317201829
  -486> 2017-11-05 21:59:30.773969 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
  -485> 2017-11-05 21:59:30.773972 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(auth 26252..26332) dispatch 0x7f2342f14500 auth(proto 2 165 bytes epoch 0) v1 from client.? 10.10.1.21:0/1317201829 con 0x7f2342f07800
  -484> 2017-11-05 21:59:30.773979 7f23302e4700  5 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907754) is_readable = 1 - now=2017-11-05 21:59:30.773979 lease_expire=2017-11-05 21:59:35.342590 has v0 lc 112907754
  -483> 2017-11-05 21:59:30.773988 7f23302e4700 10 mon.ceph03@2(peon).auth v26332 preprocess_query auth(proto 2 165 bytes epoch 0) v1 from client.? 10.10.1.21:0/1317201829
  -482> 2017-11-05 21:59:30.773995 7f23302e4700 10 mon.ceph03@2(peon).auth v26332 prep_auth() blob_size=165
  -481> 2017-11-05 21:59:30.773999 7f23302e4700 10 cephx server client.admin: handle_request get_principal_session_key
  -480> 2017-11-05 21:59:30.774002 7f23302e4700 10 cephx: verify_authorizer decrypted service auth secret_id=1699
  -479> 2017-11-05 21:59:30.774054 7f23302e4700 10 cephx: verify_authorizer global_id=55234103
  -478> 2017-11-05 21:59:30.774080 7f23302e4700 10 cephx: verify_authorizer ok nonce 643c986966334873 reply_bl.length()=36
  -477> 2017-11-05 21:59:30.774083 7f23302e4700 10 cephx server client.admin:  ticket_req.keys = 21
  -476> 2017-11-05 21:59:30.774085 7f23302e4700 10 cephx server client.admin:  adding key for service mon
  -475> 2017-11-05 21:59:30.774171 7f23302e4700 10 cephx server client.admin:  adding key for service osd
  -474> 2017-11-05 21:59:30.774207 7f23302e4700 10 cephx keyserverdata: get_caps: name=client.admin
  -473> 2017-11-05 21:59:30.774214 7f23302e4700 10 cephx keyserverdata: get_secret: num of caps=4
  -472> 2017-11-05 21:59:30.774218 7f23302e4700 10 cephx server client.admin:  adding key for service mgr
  -471> 2017-11-05 21:59:30.774252 7f23302e4700 10 cephx keyserverdata: get_caps: name=client.admin
  -470> 2017-11-05 21:59:30.774258 7f23302e4700 10 cephx keyserverdata: get_secret: num of caps=4
  -469> 2017-11-05 21:59:30.774270 7f23302e4700 10 cephx: build_service_ticket_reply encoding 3 tickets with secret AQCye/9ZNCEYLhAAM+olmzsar3Xp5cgMvcXkxg==
  -468> 2017-11-05 21:59:30.774296 7f23302e4700 10 cephx: build_service_ticket service mon secret_id 20359 ticket_info.ticket.name=client.admin
  -467> 2017-11-05 21:59:30.774323 7f23302e4700 10 cephx: build_service_ticket service osd secret_id 20359 ticket_info.ticket.name=client.admin
  -466> 2017-11-05 21:59:30.774347 7f23302e4700 10 cephx: build_service_ticket service mgr secret_id 7 ticket_info.ticket.name=client.admin
  -465> 2017-11-05 21:59:30.774381 7f23302e4700  2 mon.ceph03@2(peon) e14 send_reply 0x7f2342e9d4e0 0x7f234296d900 auth_reply(proto 2 0 (0) Success) v1
  -464> 2017-11-05 21:59:30.774398 7f23302e4700  1 -- 10.10.1.23:6789/0 --> 10.10.1.21:0/1317201829 -- auth_reply(proto 2 0 (0) Success) v1 -- 0x7f234296d900 con 0
  -463> 2017-11-05 21:59:30.774463 7f232d2de700  1 -- 10.10.1.23:6789/0 >> 10.10.1.21:0/1317201829 conn(0x7f2342f07800 :6789 s=STATE_OPEN pgs=2 cs=1 l=1).read_bulk peer close file descriptor 37
  -462> 2017-11-05 21:59:30.774490 7f232d2de700  1 -- 10.10.1.23:6789/0 >> 10.10.1.21:0/1317201829 conn(0x7f2342f07800 :6789 s=STATE_OPEN pgs=2 cs=1 l=1).read_until read failed
  -461> 2017-11-05 21:59:30.774499 7f232d2de700  1 -- 10.10.1.23:6789/0 >> 10.10.1.21:0/1317201829 conn(0x7f2342f07800 :6789 s=STATE_OPEN pgs=2 cs=1 l=1).process read tag failed
  -460> 2017-11-05 21:59:30.774506 7f232d2de700  1 -- 10.10.1.23:6789/0 >> 10.10.1.21:0/1317201829 conn(0x7f2342f07800 :6789 s=STATE_OPEN pgs=2 cs=1 l=1).fault on lossy channel, failing
  -459> 2017-11-05 21:59:30.774514 7f232d2de700  2 -- 10.10.1.23:6789/0 >> 10.10.1.21:0/1317201829 conn(0x7f2342f07800 :6789 s=STATE_OPEN pgs=2 cs=1 l=1)._stop
  -458> 2017-11-05 21:59:30.774551 7f23302e4700 10 mon.ceph03@2(peon) e14 ms_handle_reset 0x7f2342f07800 10.10.1.21:0/1317201829
  -457> 2017-11-05 21:59:30.774563 7f23302e4700 10 mon.ceph03@2(peon) e14 reset/close on session client.? 10.10.1.21:0/1317201829
  -456> 2017-11-05 21:59:30.774568 7f23302e4700 10 mon.ceph03@2(peon) e14 remove_session 0x7f2342933600 client.? 10.10.1.21:0/1317201829 features 0x1ffddff8eea4fffb
  -455> 2017-11-05 21:59:31.374295 7f232c2dc700 10 _calc_signature seq 248515282 front_crc_ = 4013096043 middle_crc = 0 data_crc = 0 sig = 3389298517379725511
  -454> 2017-11-05 21:59:31.374317 7f232c2dc700  5 -- 10.10.1.23:6789/0 >> 10.10.1.21:6789/0 conn(0x7f2342dd4800 :-1 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=1124301 cs=1 l=0). rx mon.0 seq 248515282 0x7f2342ef2c00 paxos(begin lc 112907754 fc 0 pn 13700 opn 0) v4
  -453> 2017-11-05 21:59:31.374362 7f23302e4700  1 -- 10.10.1.23:6789/0 <== mon.0 10.10.1.21:6789/0 248515282 ==== paxos(begin lc 112907754 fc 0 pn 13700 opn 0) v4 ==== 10535+0+0 (4013096043 0 0) 0x7f2342ef2c00 con 0x7f2342dd4800
  -452> 2017-11-05 21:59:31.374391 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
  -451> 2017-11-05 21:59:31.374403 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
  -450> 2017-11-05 21:59:31.374407 7f23302e4700 20 is_capable service=mon command= read on cap allow *
  -449> 2017-11-05 21:59:31.374411 7f23302e4700 20  allow so far , doing grant allow *
  -448> 2017-11-05 21:59:31.374412 7f23302e4700 20  allow all
  -447> 2017-11-05 21:59:31.374414 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
  -446> 2017-11-05 21:59:31.374419 7f23302e4700 20  allow so far , doing grant allow *
  -445> 2017-11-05 21:59:31.374420 7f23302e4700 20  allow all
  -444> 2017-11-05 21:59:31.374430 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907754) handle_begin paxos(begin lc 112907754 fc 0 pn 13700 opn 0) v4
  -443> 2017-11-05 21:59:31.374436 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos updating c 112907080..112907754) accepting value for 112907755 pn 13700
  -442> 2017-11-05 21:59:31.411695 7f23302e4700  1 -- 10.10.1.23:6789/0 --> 10.10.1.21:6789/0 -- paxos(accept lc 112907754 fc 0 pn 13700 opn 0) v4 -- 0x7f2342ef2900 con 0
  -441> 2017-11-05 21:59:31.411790 7f232c2dc700 10 _calc_signature seq 44 front_crc_ = 1766181624 middle_crc = 0 data_crc = 0 sig = 15812918676843081859
  -440> 2017-11-05 21:59:31.411807 7f232c2dc700 20 Putting signature in client message(seq # 44): sig = 15812918676843081859
  -439> 2017-11-05 21:59:31.440764 7f232c2dc700 10 _calc_signature seq 248515283 front_crc_ = 4114550218 middle_crc = 0 data_crc = 0 sig = 10113114477147680219
  -438> 2017-11-05 21:59:31.440781 7f232c2dc700  5 -- 10.10.1.23:6789/0 >> 10.10.1.21:6789/0 conn(0x7f2342dd4800 :-1 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=1124301 cs=1 l=0). rx mon.0 seq 248515283 0x7f2342ef2900 paxos(commit lc 112907755 fc 0 pn 13700 opn 0) v4
  -437> 2017-11-05 21:59:31.440818 7f23302e4700  1 -- 10.10.1.23:6789/0 <== mon.0 10.10.1.21:6789/0 248515283 ==== paxos(commit lc 112907755 fc 0 pn 13700 opn 0) v4 ==== 10535+0+0 (4114550218 0 0) 0x7f2342ef2900 con 0x7f2342dd4800
  -436> 2017-11-05 21:59:31.440844 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
  -435> 2017-11-05 21:59:31.440857 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
  -434> 2017-11-05 21:59:31.440861 7f23302e4700 20 is_capable service=mon command= read on cap allow *
  -433> 2017-11-05 21:59:31.440863 7f23302e4700 20  allow so far , doing grant allow *
  -432> 2017-11-05 21:59:31.440865 7f23302e4700 20  allow all
  -431> 2017-11-05 21:59:31.440867 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
  -430> 2017-11-05 21:59:31.440868 7f23302e4700 20  allow so far , doing grant allow *
  -429> 2017-11-05 21:59:31.440869 7f23302e4700 20  allow all
  -428> 2017-11-05 21:59:31.440874 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos updating c 112907080..112907754) handle_commit on 112907755
  -427> 2017-11-05 21:59:31.440879 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos updating c 112907080..112907755) store_state [112907755..112907755]
  -426> 2017-11-05 21:59:31.441178 7f232c2dc700 10 _calc_signature seq 248515284 front_crc_ = 843291185 middle_crc = 0 data_crc = 0 sig = 2691046791697793914
  -425> 2017-11-05 21:59:31.441194 7f232c2dc700  5 -- 10.10.1.23:6789/0 >> 10.10.1.21:6789/0 conn(0x7f2342dd4800 :-1 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=1124301 cs=1 l=0). rx mon.0 seq 248515284 0x7f2342ef2600 paxos(lease lc 112907755 fc 112907080 pn 0 opn 0) v4
  -424> 2017-11-05 21:59:31.478345 7f23302e4700 10 mon.ceph03@2(peon) e14 refresh_from_paxos
  -423> 2017-11-05 21:59:31.478382 7f23302e4700 10 mon.ceph03@2(peon) e14 refresh_from_paxos no cluster_fingerprint
  -422> 2017-11-05 21:59:31.478418 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(pgmap 56600067..56600783) refresh
  -421> 2017-11-05 21:59:31.478464 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mdsmap 1..1) refresh
  -420> 2017-11-05 21:59:31.478514 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(osdmap 503..1105) refresh
  -419> 2017-11-05 21:59:31.478552 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(logm 55596915..55597638) refresh
  -418> 2017-11-05 21:59:31.478559 7f23302e4700 10 mon.ceph03@2(peon).log v55597638 update_from_paxos
  -417> 2017-11-05 21:59:31.478562 7f23302e4700 10 mon.ceph03@2(peon).log v55597638 update_from_paxos version 55597638 summary v 55597637
  -416> 2017-11-05 21:59:31.478572 7f23302e4700 10 mon.ceph03@2(peon).log v55597638 update_from_paxos latest full 55597637
  -415> 2017-11-05 21:59:31.478596 7f23302e4700  7 mon.ceph03@2(peon).log v55597638 update_from_paxos applying incremental log 55597638 2017-11-05 21:59:30.880506 mon.ceph01mon mon.0 10.10.1.21:6789/0 867 : audit [DBG] from='client.? 10.10.1.21:0/1168097192' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch
  -414> 2017-11-05 21:59:31.478620 7f23302e4700 20 mon.ceph03@2(peon).log v55597638 update_from_paxos logging for channel 'audit' to file '/var/log/ceph/ceph.audit.log'
  -413> 2017-11-05 21:59:31.478658 7f23302e4700  7 mon.ceph03@2(peon).log v55597638 update_from_paxos applying incremental log 55597638 2017-11-05 21:59:30.929533 mon.ceph02mon mon.1 10.10.1.22:6789/0 363 : audit [DBG] from='client.? 10.10.1.21:0/1317201829' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch
  -412> 2017-11-05 21:59:31.478674 7f23302e4700 20 mon.ceph03@2(peon).log v55597638 update_from_paxos logging for channel 'audit' to file '/var/log/ceph/ceph.audit.log'
  -411> 2017-11-05 21:59:31.478682 7f23302e4700  7 mon.ceph03@2(peon).log v55597638 update_from_paxos applying incremental log 55597638 2017-11-05 21:59:31.014549 mon.ceph01mon mon.0 10.10.1.21:6789/0 868 : audit [DBG] from='client.? 10.10.1.21:0/1003889937' entity='client.admin' cmd=[{"prefix": "df", "format": "json"}]: dispatch
  -410> 2017-11-05 21:59:31.478689 7f23302e4700 20 mon.ceph03@2(peon).log v55597638 update_from_paxos logging for channel 'audit' to file '/var/log/ceph/ceph.audit.log'
  -409> 2017-11-05 21:59:31.478702 7f23302e4700 15 mon.ceph03@2(peon).log v55597638 update_from_paxos logging for 0 channels
  -408> 2017-11-05 21:59:31.478709 7f23302e4700 10 mon.ceph03@2(peon).log v55597638 check_subs
  -407> 2017-11-05 21:59:31.478762 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(monmap 1..14) refresh
  -406> 2017-11-05 21:59:31.478824 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(auth 26252..26332) refresh
  -405> 2017-11-05 21:59:31.478832 7f23302e4700 10 mon.ceph03@2(peon).auth v26332 update_from_paxos
  -404> 2017-11-05 21:59:31.478881 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgr 1..394) refresh
  -403> 2017-11-05 21:59:31.478895 7f23302e4700  4 mgrc handle_mgr_map Got map version 394
  -402> 2017-11-05 21:59:31.478898 7f23302e4700  4 mgrc handle_mgr_map Active mgr is now 10.10.1.21:6806/22556
  -401> 2017-11-05 21:59:31.478938 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgrstat 56609513..56609643) refresh
  -400> 2017-11-05 21:59:31.478945 7f23302e4700 10 mon.ceph03@2(peon).mgrstat  56609643
  -399> 2017-11-05 21:59:31.479003 7f23302e4700 10 mon.ceph03@2(peon).mgrstat update_from_paxos v56609643 service_map e1
  -398> 2017-11-05 21:59:31.479017 7f23302e4700 10 mon.ceph03@2(peon).mgrstat check_subs
  -397> 2017-11-05 21:59:31.479020 7f23302e4700 20 mon.ceph03@2(peon).mgrstat update_logger
  -396> 2017-11-05 21:59:31.479056 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(health 1..24) refresh
  -395> 2017-11-05 21:59:31.479067 7f23302e4700 10 mon.ceph03@2(peon).health update_from_paxos
  -394> 2017-11-05 21:59:31.479084 7f23302e4700 20 mon.ceph03@2(peon).health dump:{
    "quorum_health": {},
    "leader_health": {}
}

  -393> 2017-11-05 21:59:31.479100 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(pgmap 56600067..56600783) post_refresh
  -392> 2017-11-05 21:59:31.479105 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mdsmap 1..1) post_refresh
  -391> 2017-11-05 21:59:31.479107 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(osdmap 503..1105) post_refresh
  -390> 2017-11-05 21:59:31.479109 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(logm 55596915..55597638) post_refresh
  -389> 2017-11-05 21:59:31.479111 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(monmap 1..14) post_refresh
  -388> 2017-11-05 21:59:31.479112 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(auth 26252..26332) post_refresh
  -387> 2017-11-05 21:59:31.479114 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgr 1..394) post_refresh
  -386> 2017-11-05 21:59:31.479116 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgrstat 56609513..56609643) post_refresh
  -385> 2017-11-05 21:59:31.479117 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(health 1..24) post_refresh
  -384> 2017-11-05 21:59:31.479197 7f23302e4700  1 -- 10.10.1.23:6789/0 <== mon.0 10.10.1.21:6789/0 248515284 ==== paxos(lease lc 112907755 fc 112907080 pn 0 opn 0) v4 ==== 84+0+0 (843291185 0 0) 0x7f2342ef2600 con 0x7f2342dd4800
  -383> 2017-11-05 21:59:31.479217 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
  -382> 2017-11-05 21:59:31.479226 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
  -381> 2017-11-05 21:59:31.479229 7f23302e4700 20 is_capable service=mon command= read on cap allow *
  -380> 2017-11-05 21:59:31.479232 7f23302e4700 20  allow so far , doing grant allow *
  -379> 2017-11-05 21:59:31.479233 7f23302e4700 20  allow all
  -378> 2017-11-05 21:59:31.479235 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
  -377> 2017-11-05 21:59:31.479236 7f23302e4700 20  allow so far , doing grant allow *
  -376> 2017-11-05 21:59:31.479237 7f23302e4700 20  allow all
  -375> 2017-11-05 21:59:31.479242 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907755) handle_lease on 112907755 now 2017-11-05 21:59:36.439247
  -374> 2017-11-05 21:59:31.479253 7f23302e4700  1 -- 10.10.1.23:6789/0 --> 10.10.1.21:6789/0 -- paxos(lease_ack lc 112907755 fc 112907080 pn 0 opn 0) v4 -- 0x7f2342ef2900 con 0
  -373> 2017-11-05 21:59:31.479272 7f23302e4700 20 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907755) reset_lease_timeout - setting timeout event
  -372> 2017-11-05 21:59:31.479280 7f23302e4700  5 mon.ceph03@2(peon).monmap v14 apply_mon_features wait for service to be writeable
  -371> 2017-11-05 21:59:31.479283 7f23302e4700  5 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907755) is_readable = 1 - now=2017-11-05 21:59:31.479284 lease_expire=2017-11-05 21:59:36.439247 has v0 lc 112907755
  -370> 2017-11-05 21:59:31.479338 7f232c2dc700 10 _calc_signature seq 45 front_crc_ = 3341093255 middle_crc = 0 data_crc = 0 sig = 199787768109639415
  -369> 2017-11-05 21:59:31.479354 7f232c2dc700 20 Putting signature in client message(seq # 45): sig = 199787768109639415
  -368> 2017-11-05 21:59:32.481784 7f232c2dc700 10 _calc_signature seq 248515285 front_crc_ = 2658085271 middle_crc = 0 data_crc = 0 sig = 15694764974702500250
  -367> 2017-11-05 21:59:32.481805 7f232c2dc700  5 -- 10.10.1.23:6789/0 >> 10.10.1.21:6789/0 conn(0x7f2342dd4800 :-1 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=1124301 cs=1 l=0). rx mon.0 seq 248515285 0x7f2342ef2900 paxos(begin lc 112907755 fc 0 pn 13700 opn 0) v4
  -366> 2017-11-05 21:59:32.481869 7f23302e4700  1 -- 10.10.1.23:6789/0 <== mon.0 10.10.1.21:6789/0 248515285 ==== paxos(begin lc 112907755 fc 0 pn 13700 opn 0) v4 ==== 3332+0+0 (2658085271 0 0) 0x7f2342ef2900 con 0x7f2342dd4800
  -365> 2017-11-05 21:59:32.481906 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
  -364> 2017-11-05 21:59:32.481916 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
  -363> 2017-11-05 21:59:32.481921 7f23302e4700 20 is_capable service=mon command= read on cap allow *
  -362> 2017-11-05 21:59:32.481924 7f23302e4700 20  allow so far , doing grant allow *
  -361> 2017-11-05 21:59:32.481925 7f23302e4700 20  allow all
  -360> 2017-11-05 21:59:32.481930 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
  -359> 2017-11-05 21:59:32.481932 7f23302e4700 20  allow so far , doing grant allow *
  -358> 2017-11-05 21:59:32.481933 7f23302e4700 20  allow all
  -357> 2017-11-05 21:59:32.481939 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907755) handle_begin paxos(begin lc 112907755 fc 0 pn 13700 opn 0) v4
  -356> 2017-11-05 21:59:32.481944 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos updating c 112907080..112907755) accepting value for 112907756 pn 13700
  -355> 2017-11-05 21:59:32.519124 7f23302e4700  1 -- 10.10.1.23:6789/0 --> 10.10.1.21:6789/0 -- paxos(accept lc 112907755 fc 0 pn 13700 opn 0) v4 -- 0x7f2342ef2600 con 0
  -354> 2017-11-05 21:59:32.519206 7f232c2dc700 10 _calc_signature seq 46 front_crc_ = 916056675 middle_crc = 0 data_crc = 0 sig = 3319636971414846998
  -353> 2017-11-05 21:59:32.519222 7f232c2dc700 20 Putting signature in client message(seq # 46): sig = 3319636971414846998
  -352> 2017-11-05 21:59:32.548470 7f232c2dc700 10 _calc_signature seq 248515286 front_crc_ = 3380664477 middle_crc = 0 data_crc = 0 sig = 16252625642398161639
  -351> 2017-11-05 21:59:32.548486 7f232c2dc700  5 -- 10.10.1.23:6789/0 >> 10.10.1.21:6789/0 conn(0x7f2342dd4800 :-1 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=1124301 cs=1 l=0). rx mon.0 seq 248515286 0x7f2342ef2600 paxos(commit lc 112907756 fc 0 pn 13700 opn 0) v4
  -350> 2017-11-05 21:59:32.548545 7f23302e4700  1 -- 10.10.1.23:6789/0 <== mon.0 10.10.1.21:6789/0 248515286 ==== paxos(commit lc 112907756 fc 0 pn 13700 opn 0) v4 ==== 3332+0+0 (3380664477 0 0) 0x7f2342ef2600 con 0x7f2342dd4800
  -349> 2017-11-05 21:59:32.548580 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
  -348> 2017-11-05 21:59:32.548589 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
  -347> 2017-11-05 21:59:32.548592 7f23302e4700 20 is_capable service=mon command= read on cap allow *
  -346> 2017-11-05 21:59:32.548595 7f23302e4700 20  allow so far , doing grant allow *
  -345> 2017-11-05 21:59:32.548596 7f23302e4700 20  allow all
  -344> 2017-11-05 21:59:32.548598 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
  -343> 2017-11-05 21:59:32.548600 7f23302e4700 20  allow so far , doing grant allow *
  -342> 2017-11-05 21:59:32.548601 7f23302e4700 20  allow all
  -341> 2017-11-05 21:59:32.548606 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos updating c 112907080..112907755) handle_commit on 112907756
  -340> 2017-11-05 21:59:32.548611 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos updating c 112907080..112907756) store_state [112907756..112907756]
  -339> 2017-11-05 21:59:32.548902 7f232c2dc700 10 _calc_signature seq 248515287 front_crc_ = 2140225877 middle_crc = 0 data_crc = 0 sig = 17236814342560090484
  -338> 2017-11-05 21:59:32.548916 7f232c2dc700  5 -- 10.10.1.23:6789/0 >> 10.10.1.21:6789/0 conn(0x7f2342dd4800 :-1 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=1124301 cs=1 l=0). rx mon.0 seq 248515287 0x7f2342ef2300 paxos(lease lc 112907756 fc 112907080 pn 0 opn 0) v4
  -337> 2017-11-05 21:59:32.581973 7f232c2dc700 10 _calc_signature seq 248515288 front_crc_ = 596978514 middle_crc = 0 data_crc = 0 sig = 17084270481735144864
  -336> 2017-11-05 21:59:32.581990 7f232c2dc700  5 -- 10.10.1.23:6789/0 >> 10.10.1.21:6789/0 conn(0x7f2342dd4800 :-1 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=1124301 cs=1 l=0). rx mon.0 seq 248515288 0x7f2342ef2000 paxos(begin lc 112907756 fc 0 pn 13700 opn 0) v4
  -335> 2017-11-05 21:59:32.585752 7f23302e4700 10 mon.ceph03@2(peon) e14 refresh_from_paxos
  -334> 2017-11-05 21:59:32.585791 7f23302e4700 10 mon.ceph03@2(peon) e14 refresh_from_paxos no cluster_fingerprint
  -333> 2017-11-05 21:59:32.585826 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(pgmap 56600067..56600783) refresh
  -332> 2017-11-05 21:59:32.585878 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mdsmap 1..1) refresh
  -331> 2017-11-05 21:59:32.585937 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(osdmap 503..1105) refresh
  -330> 2017-11-05 21:59:32.585980 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(logm 55596915..55597638) refresh
  -329> 2017-11-05 21:59:32.585987 7f23302e4700 10 mon.ceph03@2(peon).log v55597638 update_from_paxos
  -328> 2017-11-05 21:59:32.585990 7f23302e4700 10 mon.ceph03@2(peon).log v55597638 update_from_paxos version 55597638 summary v 55597638
  -327> 2017-11-05 21:59:32.586037 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(monmap 1..14) refresh
  -326> 2017-11-05 21:59:32.586064 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(auth 26252..26332) refresh
  -325> 2017-11-05 21:59:32.586070 7f23302e4700 10 mon.ceph03@2(peon).auth v26332 update_from_paxos
  -324> 2017-11-05 21:59:32.586118 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgr 1..394) refresh
  -323> 2017-11-05 21:59:32.586154 7f23302e4700  4 mgrc handle_mgr_map Got map version 394
  -322> 2017-11-05 21:59:32.586158 7f23302e4700  4 mgrc handle_mgr_map Active mgr is now 10.10.1.21:6806/22556
  -321> 2017-11-05 21:59:32.586206 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgrstat 56609513..56609644) refresh
  -320> 2017-11-05 21:59:32.586215 7f23302e4700 10 mon.ceph03@2(peon).mgrstat  56609644
  -319> 2017-11-05 21:59:32.586264 7f23302e4700 10 mon.ceph03@2(peon).mgrstat update_from_paxos v56609644 service_map e1
  -318> 2017-11-05 21:59:32.586274 7f23302e4700 10 mon.ceph03@2(peon).mgrstat check_subs
  -317> 2017-11-05 21:59:32.586276 7f23302e4700 20 mon.ceph03@2(peon).mgrstat update_logger
  -316> 2017-11-05 21:59:32.586310 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(health 1..24) refresh
  -315> 2017-11-05 21:59:32.586317 7f23302e4700 10 mon.ceph03@2(peon).health update_from_paxos
  -314> 2017-11-05 21:59:32.586337 7f23302e4700 20 mon.ceph03@2(peon).health dump:{
    "quorum_health": {},
    "leader_health": {}
}

  -313> 2017-11-05 21:59:32.586353 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(pgmap 56600067..56600783) post_refresh
  -312> 2017-11-05 21:59:32.586356 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mdsmap 1..1) post_refresh
  -311> 2017-11-05 21:59:32.586359 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(osdmap 503..1105) post_refresh
  -310> 2017-11-05 21:59:32.586360 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(logm 55596915..55597638) post_refresh
  -309> 2017-11-05 21:59:32.586362 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(monmap 1..14) post_refresh
  -308> 2017-11-05 21:59:32.586364 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(auth 26252..26332) post_refresh
  -307> 2017-11-05 21:59:32.586365 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgr 1..394) post_refresh
  -306> 2017-11-05 21:59:32.586367 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgrstat 56609513..56609644) post_refresh
  -305> 2017-11-05 21:59:32.586369 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(health 1..24) post_refresh
  -304> 2017-11-05 21:59:32.586454 7f23302e4700  1 -- 10.10.1.23:6789/0 <== mon.0 10.10.1.21:6789/0 248515287 ==== paxos(lease lc 112907756 fc 112907080 pn 0 opn 0) v4 ==== 84+0+0 (2140225877 0 0) 0x7f2342ef2300 con 0x7f2342dd4800
  -303> 2017-11-05 21:59:32.586474 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
  -302> 2017-11-05 21:59:32.586486 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
  -301> 2017-11-05 21:59:32.586489 7f23302e4700 20 is_capable service=mon command= read on cap allow *
  -300> 2017-11-05 21:59:32.586492 7f23302e4700 20  allow so far , doing grant allow *
  -299> 2017-11-05 21:59:32.586493 7f23302e4700 20  allow all
  -298> 2017-11-05 21:59:32.586495 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
  -297> 2017-11-05 21:59:32.586497 7f23302e4700 20  allow so far , doing grant allow *
  -296> 2017-11-05 21:59:32.586498 7f23302e4700 20  allow all
  -295> 2017-11-05 21:59:32.586502 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907756) handle_lease on 112907756 now 2017-11-05 21:59:37.547047
  -294> 2017-11-05 21:59:32.586518 7f23302e4700  1 -- 10.10.1.23:6789/0 --> 10.10.1.21:6789/0 -- paxos(lease_ack lc 112907756 fc 112907080 pn 0 opn 0) v4 -- 0x7f2342ef2600 con 0
  -293> 2017-11-05 21:59:32.586540 7f23302e4700 20 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907756) reset_lease_timeout - setting timeout event
  -292> 2017-11-05 21:59:32.586547 7f23302e4700  5 mon.ceph03@2(peon).monmap v14 apply_mon_features wait for service to be writeable
  -291> 2017-11-05 21:59:32.586551 7f23302e4700  5 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907756) is_readable = 1 - now=2017-11-05 21:59:32.586551 lease_expire=2017-11-05 21:59:37.547047 has v0 lc 112907756
  -290> 2017-11-05 21:59:32.586564 7f23302e4700  1 -- 10.10.1.23:6789/0 <== mon.0 10.10.1.21:6789/0 248515288 ==== paxos(begin lc 112907756 fc 0 pn 13700 opn 0) v4 ==== 10123+0+0 (596978514 0 0) 0x7f2342ef2000 con 0x7f2342dd4800
  -289> 2017-11-05 21:59:32.586577 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
  -288> 2017-11-05 21:59:32.586586 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
  -287> 2017-11-05 21:59:32.586588 7f23302e4700 20 is_capable service=mon command= read on cap allow *
  -286> 2017-11-05 21:59:32.586590 7f23302e4700 20  allow so far , doing grant allow *
  -285> 2017-11-05 21:59:32.586591 7f23302e4700 20  allow all
  -284> 2017-11-05 21:59:32.586586 7f232c2dc700 10 _calc_signature seq 47 front_crc_ = 514726373 middle_crc = 0 data_crc = 0 sig = 14918821708796830444
  -283> 2017-11-05 21:59:32.586592 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
  -282> 2017-11-05 21:59:32.586594 7f23302e4700 20  allow so far , doing grant allow *
  -281> 2017-11-05 21:59:32.586594 7f232c2dc700 20 Putting signature in client message(seq # 47): sig = 14918821708796830444
  -280> 2017-11-05 21:59:32.586596 7f23302e4700 20  allow all
  -279> 2017-11-05 21:59:32.586599 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907756) handle_begin paxos(begin lc 112907756 fc 0 pn 13700 opn 0) v4
  -278> 2017-11-05 21:59:32.586619 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos updating c 112907080..112907756) accepting value for 112907757 pn 13700
  -277> 2017-11-05 21:59:32.630155 7f23302e4700  1 -- 10.10.1.23:6789/0 --> 10.10.1.21:6789/0 -- paxos(accept lc 112907756 fc 0 pn 13700 opn 0) v4 -- 0x7f2342ef2300 con 0
  -276> 2017-11-05 21:59:32.630246 7f232c2dc700 10 _calc_signature seq 48 front_crc_ = 3355026546 middle_crc = 0 data_crc = 0 sig = 10281745577978676838
  -275> 2017-11-05 21:59:32.630261 7f232c2dc700 20 Putting signature in client message(seq # 48): sig = 10281745577978676838
  -274> 2017-11-05 21:59:32.659832 7f232c2dc700 10 _calc_signature seq 248515289 front_crc_ = 3094126013 middle_crc = 0 data_crc = 0 sig = 18218305787367258982
  -273> 2017-11-05 21:59:32.659850 7f232c2dc700  5 -- 10.10.1.23:6789/0 >> 10.10.1.21:6789/0 conn(0x7f2342dd4800 :-1 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=1124301 cs=1 l=0). rx mon.0 seq 248515289 0x7f2342ef2300 paxos(commit lc 112907757 fc 0 pn 13700 opn 0) v4
  -272> 2017-11-05 21:59:32.659911 7f23302e4700  1 -- 10.10.1.23:6789/0 <== mon.0 10.10.1.21:6789/0 248515289 ==== paxos(commit lc 112907757 fc 0 pn 13700 opn 0) v4 ==== 10123+0+0 (3094126013 0 0) 0x7f2342ef2300 con 0x7f2342dd4800
  -271> 2017-11-05 21:59:32.659945 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
  -270> 2017-11-05 21:59:32.659957 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
  -269> 2017-11-05 21:59:32.659960 7f23302e4700 20 is_capable service=mon command= read on cap allow *
  -268> 2017-11-05 21:59:32.659963 7f23302e4700 20  allow so far , doing grant allow *
  -267> 2017-11-05 21:59:32.659964 7f23302e4700 20  allow all
  -266> 2017-11-05 21:59:32.659966 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
  -265> 2017-11-05 21:59:32.659967 7f23302e4700 20  allow so far , doing grant allow *
  -264> 2017-11-05 21:59:32.659969 7f23302e4700 20  allow all
  -263> 2017-11-05 21:59:32.659974 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos updating c 112907080..112907756) handle_commit on 112907757
  -262> 2017-11-05 21:59:32.659979 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos updating c 112907080..112907757) store_state [112907757..112907757]
  -261> 2017-11-05 21:59:32.660040 7f232c2dc700 10 _calc_signature seq 248515290 front_crc_ = 2812660264 middle_crc = 0 data_crc = 0 sig = 6326750952372116244
  -260> 2017-11-05 21:59:32.660046 7f232c2dc700  5 -- 10.10.1.23:6789/0 >> 10.10.1.21:6789/0 conn(0x7f2342dd4800 :-1 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=1124301 cs=1 l=0). rx mon.0 seq 248515290 0x7f2342ef2600 paxos(lease lc 112907757 fc 112907080 pn 0 opn 0) v4
  -259> 2017-11-05 21:59:32.696757 7f23302e4700 10 mon.ceph03@2(peon) e14 refresh_from_paxos
  -258> 2017-11-05 21:59:32.696792 7f23302e4700 10 mon.ceph03@2(peon) e14 refresh_from_paxos no cluster_fingerprint
  -257> 2017-11-05 21:59:32.696839 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(pgmap 56600067..56600783) refresh
  -256> 2017-11-05 21:59:32.696888 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mdsmap 1..1) refresh
  -255> 2017-11-05 21:59:32.696935 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(osdmap 503..1105) refresh
  -254> 2017-11-05 21:59:32.696977 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(logm 55596915..55597639) refresh
  -253> 2017-11-05 21:59:32.696984 7f23302e4700 10 mon.ceph03@2(peon).log v55597639 update_from_paxos
  -252> 2017-11-05 21:59:32.696986 7f23302e4700 10 mon.ceph03@2(peon).log v55597639 update_from_paxos version 55597639 summary v 55597638
  -251> 2017-11-05 21:59:32.696997 7f23302e4700 10 mon.ceph03@2(peon).log v55597639 update_from_paxos latest full 55597638
  -250> 2017-11-05 21:59:32.697035 7f23302e4700  7 mon.ceph03@2(peon).log v55597639 update_from_paxos applying incremental log 55597639 2017-11-05 21:59:30.742011 mgr.ceph01mgr client.54805773 10.10.1.21:0/1619113657 1819 : audit [DBG] from='client.55026414 10.10.1.21:0/3473720941' entity='client.admin' cmd=[{"prefix": "osd pool stats", "target": ["mgr", ""], "format": "json"}]: dispatch
  -249> 2017-11-05 21:59:32.697059 7f23302e4700 20 mon.ceph03@2(peon).log v55597639 update_from_paxos logging for channel 'audit' to file '/var/log/ceph/ceph.audit.log'
  -248> 2017-11-05 21:59:32.697081 7f23302e4700 15 mon.ceph03@2(peon).log v55597639 update_from_paxos logging for 0 channels
  -247> 2017-11-05 21:59:32.697087 7f23302e4700 10 mon.ceph03@2(peon).log v55597639 check_subs
  -246> 2017-11-05 21:59:32.697146 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(monmap 1..14) refresh
  -245> 2017-11-05 21:59:32.697182 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(auth 26252..26332) refresh
  -244> 2017-11-05 21:59:32.697188 7f23302e4700 10 mon.ceph03@2(peon).auth v26332 update_from_paxos
  -243> 2017-11-05 21:59:32.697242 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgr 1..394) refresh
  -242> 2017-11-05 21:59:32.697253 7f23302e4700  4 mgrc handle_mgr_map Got map version 394
  -241> 2017-11-05 21:59:32.697256 7f23302e4700  4 mgrc handle_mgr_map Active mgr is now 10.10.1.21:6806/22556
  -240> 2017-11-05 21:59:32.697302 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgrstat 56609513..56609644) refresh
  -239> 2017-11-05 21:59:32.697332 7f23302e4700 10 mon.ceph03@2(peon).mgrstat  56609644
  -238> 2017-11-05 21:59:32.697393 7f23302e4700 10 mon.ceph03@2(peon).mgrstat update_from_paxos v56609644 service_map e1
  -237> 2017-11-05 21:59:32.697400 7f23302e4700 10 mon.ceph03@2(peon).mgrstat check_subs
  -236> 2017-11-05 21:59:32.697402 7f23302e4700 20 mon.ceph03@2(peon).mgrstat update_logger
  -235> 2017-11-05 21:59:32.697447 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(health 1..24) refresh
  -234> 2017-11-05 21:59:32.697454 7f23302e4700 10 mon.ceph03@2(peon).health update_from_paxos
  -233> 2017-11-05 21:59:32.697480 7f23302e4700 20 mon.ceph03@2(peon).health dump:{
    "quorum_health": {},
    "leader_health": {}
}

  -232> 2017-11-05 21:59:32.697494 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(pgmap 56600067..56600783) post_refresh
  -231> 2017-11-05 21:59:32.697496 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mdsmap 1..1) post_refresh
  -230> 2017-11-05 21:59:32.697498 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(osdmap 503..1105) post_refresh
  -229> 2017-11-05 21:59:32.697500 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(logm 55596915..55597639) post_refresh
  -228> 2017-11-05 21:59:32.697502 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(monmap 1..14) post_refresh
  -227> 2017-11-05 21:59:32.697503 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(auth 26252..26332) post_refresh
  -226> 2017-11-05 21:59:32.697505 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgr 1..394) post_refresh
  -225> 2017-11-05 21:59:32.697506 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgrstat 56609513..56609644) post_refresh
  -224> 2017-11-05 21:59:32.697508 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(health 1..24) post_refresh
  -223> 2017-11-05 21:59:32.697597 7f23302e4700  1 -- 10.10.1.23:6789/0 <== mon.0 10.10.1.21:6789/0 248515290 ==== paxos(lease lc 112907757 fc 112907080 pn 0 opn 0) v4 ==== 84+0+0 (2812660264 0 0) 0x7f2342ef2600 con 0x7f2342dd4800
  -222> 2017-11-05 21:59:32.697619 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
  -221> 2017-11-05 21:59:32.697628 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
  -220> 2017-11-05 21:59:32.697631 7f23302e4700 20 is_capable service=mon command= read on cap allow *
  -219> 2017-11-05 21:59:32.697634 7f23302e4700 20  allow so far , doing grant allow *
  -218> 2017-11-05 21:59:32.697635 7f23302e4700 20  allow all
  -217> 2017-11-05 21:59:32.697637 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
  -216> 2017-11-05 21:59:32.697638 7f23302e4700 20  allow so far , doing grant allow *
  -215> 2017-11-05 21:59:32.697639 7f23302e4700 20  allow all
  -214> 2017-11-05 21:59:32.697644 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907757) handle_lease on 112907757 now 2017-11-05 21:59:37.658131
  -213> 2017-11-05 21:59:32.697654 7f23302e4700  1 -- 10.10.1.23:6789/0 --> 10.10.1.21:6789/0 -- paxos(lease_ack lc 112907757 fc 112907080 pn 0 opn 0) v4 -- 0x7f2342ef2300 con 0
  -212> 2017-11-05 21:59:32.697676 7f23302e4700 20 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907757) reset_lease_timeout - setting timeout event
  -211> 2017-11-05 21:59:32.697682 7f23302e4700  5 mon.ceph03@2(peon).monmap v14 apply_mon_features wait for service to be writeable
  -210> 2017-11-05 21:59:32.697685 7f23302e4700  5 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907757) is_readable = 1 - now=2017-11-05 21:59:32.697685 lease_expire=2017-11-05 21:59:37.658131 has v0 lc 112907757
  -209> 2017-11-05 21:59:32.697709 7f2332ae9700 11 mon.ceph03@2(peon) e14 tick
  -208> 2017-11-05 21:59:32.697722 7f232c2dc700 10 _calc_signature seq 49 front_crc_ = 2344320304 middle_crc = 0 data_crc = 0 sig = 13886427265203894317
  -207> 2017-11-05 21:59:32.697728 7f2332ae9700 10 mon.ceph03@2(peon).mds e1
  -206> 2017-11-05 21:59:32.697735 7f2332ae9700 10 mon.ceph03@2(peon).osd e1105 e1105: 9 total, 9 up, 9 in
  -205> 2017-11-05 21:59:32.697740 7f2332ae9700 10 mon.ceph03@2(peon).log v55597639 log
  -204> 2017-11-05 21:59:32.697743 7f2332ae9700 10 mon.ceph03@2(peon).auth v26332 auth
  -203> 2017-11-05 21:59:32.697732 7f232c2dc700 20 Putting signature in client message(seq # 49): sig = 13886427265203894317
  -202> 2017-11-05 21:59:32.697746 7f2332ae9700 10 mon.ceph03@2(peon).health tick
  -201> 2017-11-05 21:59:32.697748 7f2332ae9700 20 mon.ceph03@2(peon).health check_member_health
  -200> 2017-11-05 21:59:32.697826 7f2332ae9700 10 mon.ceph03@2(peon).health check_member_health avail 92% total 277 GB, used 5677 MB, avail 257 GB
  -199> 2017-11-05 21:59:32.697836 7f2332ae9700 20 mon.ceph03@2(peon) e14 sync_trim_providers
  -198> 2017-11-05 21:59:32.899066 7f232f2e2700  1 -- 10.10.1.23:0/15195 --> 10.10.1.21:6806/22556 -- mgrreport(unknown.ceph03 +0-0 packed 1734) v4 -- 0x7f2342beb080 con 0
  -197> 2017-11-05 21:59:32.899187 7f232d2de700 10 _calc_signature seq 6 front_crc_ = 3036032456 middle_crc = 0 data_crc = 0 sig = 5815314937307354885
  -196> 2017-11-05 21:59:32.899206 7f232d2de700 20 Putting signature in client message(seq # 6): sig = 5815314937307354885
  -195> 2017-11-05 21:59:34.276228 7f232c2dc700 10 _calc_signature seq 248515291 front_crc_ = 1672536857 middle_crc = 0 data_crc = 0 sig = 7744206132391408314
  -194> 2017-11-05 21:59:34.276251 7f232c2dc700  5 -- 10.10.1.23:6789/0 >> 10.10.1.21:6789/0 conn(0x7f2342dd4800 :-1 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=1124301 cs=1 l=0). rx mon.0 seq 248515291 0x7f2342ef2300 paxos(begin lc 112907757 fc 0 pn 13700 opn 0) v4
  -193> 2017-11-05 21:59:34.276343 7f23302e4700  1 -- 10.10.1.23:6789/0 <== mon.0 10.10.1.21:6789/0 248515291 ==== paxos(begin lc 112907757 fc 0 pn 13700 opn 0) v4 ==== 3316+0+0 (1672536857 0 0) 0x7f2342ef2300 con 0x7f2342dd4800
  -192> 2017-11-05 21:59:34.276380 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
  -191> 2017-11-05 21:59:34.276390 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
  -190> 2017-11-05 21:59:34.276394 7f23302e4700 20 is_capable service=mon command= read on cap allow *
  -189> 2017-11-05 21:59:34.276398 7f23302e4700 20  allow so far , doing grant allow *
  -188> 2017-11-05 21:59:34.276399 7f23302e4700 20  allow all
  -187> 2017-11-05 21:59:34.276401 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
  -186> 2017-11-05 21:59:34.276402 7f23302e4700 20  allow so far , doing grant allow *
  -185> 2017-11-05 21:59:34.276403 7f23302e4700 20  allow all
  -184> 2017-11-05 21:59:34.276409 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907757) handle_begin paxos(begin lc 112907757 fc 0 pn 13700 opn 0) v4
  -183> 2017-11-05 21:59:34.276414 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos updating c 112907080..112907757) accepting value for 112907758 pn 13700
  -182> 2017-11-05 21:59:34.313182 7f23302e4700  1 -- 10.10.1.23:6789/0 --> 10.10.1.21:6789/0 -- paxos(accept lc 112907757 fc 0 pn 13700 opn 0) v4 -- 0x7f2342ef2600 con 0
  -181> 2017-11-05 21:59:34.313293 7f232c2dc700 10 _calc_signature seq 50 front_crc_ = 25195403 middle_crc = 0 data_crc = 0 sig = 14303817380959227799
  -180> 2017-11-05 21:59:34.313312 7f232c2dc700 20 Putting signature in client message(seq # 50): sig = 14303817380959227799
  -179> 2017-11-05 21:59:34.342963 7f232c2dc700 10 _calc_signature seq 248515292 front_crc_ = 2334251510 middle_crc = 0 data_crc = 0 sig = 3881307708667676196
  -178> 2017-11-05 21:59:34.342981 7f232c2dc700  5 -- 10.10.1.23:6789/0 >> 10.10.1.21:6789/0 conn(0x7f2342dd4800 :-1 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=1124301 cs=1 l=0). rx mon.0 seq 248515292 0x7f2342ef2600 paxos(commit lc 112907758 fc 0 pn 13700 opn 0) v4
  -177> 2017-11-05 21:59:34.343042 7f23302e4700  1 -- 10.10.1.23:6789/0 <== mon.0 10.10.1.21:6789/0 248515292 ==== paxos(commit lc 112907758 fc 0 pn 13700 opn 0) v4 ==== 3316+0+0 (2334251510 0 0) 0x7f2342ef2600 con 0x7f2342dd4800
  -176> 2017-11-05 21:59:34.343077 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
  -175> 2017-11-05 21:59:34.343087 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
  -174> 2017-11-05 21:59:34.343090 7f23302e4700 20 is_capable service=mon command= read on cap allow *
  -173> 2017-11-05 21:59:34.343093 7f23302e4700 20  allow so far , doing grant allow *
  -172> 2017-11-05 21:59:34.343094 7f23302e4700 20  allow all
  -171> 2017-11-05 21:59:34.343096 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
  -170> 2017-11-05 21:59:34.343098 7f23302e4700 20  allow so far , doing grant allow *
  -169> 2017-11-05 21:59:34.343099 7f23302e4700 20  allow all
  -168> 2017-11-05 21:59:34.343104 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos updating c 112907080..112907757) handle_commit on 112907758
  -167> 2017-11-05 21:59:34.343109 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos updating c 112907080..112907758) store_state [112907758..112907758]
  -166> 2017-11-05 21:59:34.343246 7f232c2dc700 10 _calc_signature seq 248515293 front_crc_ = 3382032424 middle_crc = 0 data_crc = 0 sig = 13032620410930708538
  -165> 2017-11-05 21:59:34.343259 7f232c2dc700  5 -- 10.10.1.23:6789/0 >> 10.10.1.21:6789/0 conn(0x7f2342dd4800 :-1 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=1124301 cs=1 l=0). rx mon.0 seq 248515293 0x7f2342ef4400 paxos(lease lc 112907758 fc 112907080 pn 0 opn 0) v4
  -164> 2017-11-05 21:59:34.379800 7f23302e4700 10 mon.ceph03@2(peon) e14 refresh_from_paxos
  -163> 2017-11-05 21:59:34.379838 7f23302e4700 10 mon.ceph03@2(peon) e14 refresh_from_paxos no cluster_fingerprint
  -162> 2017-11-05 21:59:34.379874 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(pgmap 56600067..56600783) refresh
  -161> 2017-11-05 21:59:34.379922 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mdsmap 1..1) refresh
  -160> 2017-11-05 21:59:34.379973 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(osdmap 503..1105) refresh
  -159> 2017-11-05 21:59:34.380020 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(logm 55596915..55597639) refresh
  -158> 2017-11-05 21:59:34.380026 7f23302e4700 10 mon.ceph03@2(peon).log v55597639 update_from_paxos
  -157> 2017-11-05 21:59:34.380029 7f23302e4700 10 mon.ceph03@2(peon).log v55597639 update_from_paxos version 55597639 summary v 55597639
  -156> 2017-11-05 21:59:34.380077 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(monmap 1..14) refresh
  -155> 2017-11-05 21:59:34.380104 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(auth 26252..26332) refresh
  -154> 2017-11-05 21:59:34.380127 7f23302e4700 10 mon.ceph03@2(peon).auth v26332 update_from_paxos
  -153> 2017-11-05 21:59:34.380179 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgr 1..394) refresh
  -152> 2017-11-05 21:59:34.380195 7f23302e4700  4 mgrc handle_mgr_map Got map version 394
  -151> 2017-11-05 21:59:34.380199 7f23302e4700  4 mgrc handle_mgr_map Active mgr is now 10.10.1.21:6806/22556
  -150> 2017-11-05 21:59:34.380243 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgrstat 56609513..56609645) refresh
  -149> 2017-11-05 21:59:34.380250 7f23302e4700 10 mon.ceph03@2(peon).mgrstat  56609645
  -148> 2017-11-05 21:59:34.380308 7f23302e4700 10 mon.ceph03@2(peon).mgrstat update_from_paxos v56609645 service_map e1
  -147> 2017-11-05 21:59:34.380315 7f23302e4700 10 mon.ceph03@2(peon).mgrstat check_subs
  -146> 2017-11-05 21:59:34.380317 7f23302e4700 20 mon.ceph03@2(peon).mgrstat update_logger
  -145> 2017-11-05 21:59:34.380352 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(health 1..24) refresh
  -144> 2017-11-05 21:59:34.380359 7f23302e4700 10 mon.ceph03@2(peon).health update_from_paxos
  -143> 2017-11-05 21:59:34.380376 7f23302e4700 20 mon.ceph03@2(peon).health dump:{
    "quorum_health": {},
    "leader_health": {}
}

  -142> 2017-11-05 21:59:34.380392 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(pgmap 56600067..56600783) post_refresh
  -141> 2017-11-05 21:59:34.380394 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mdsmap 1..1) post_refresh
  -140> 2017-11-05 21:59:34.380399 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(osdmap 503..1105) post_refresh
  -139> 2017-11-05 21:59:34.380401 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(logm 55596915..55597639) post_refresh
  -138> 2017-11-05 21:59:34.380402 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(monmap 1..14) post_refresh
  -137> 2017-11-05 21:59:34.380404 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(auth 26252..26332) post_refresh
  -136> 2017-11-05 21:59:34.380406 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgr 1..394) post_refresh
  -135> 2017-11-05 21:59:34.380407 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(mgrstat 56609513..56609645) post_refresh
  -134> 2017-11-05 21:59:34.380409 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(health 1..24) post_refresh
  -133> 2017-11-05 21:59:34.380486 7f23302e4700  1 -- 10.10.1.23:6789/0 <== mon.0 10.10.1.21:6789/0 248515293 ==== paxos(lease lc 112907758 fc 112907080 pn 0 opn 0) v4 ==== 84+0+0 (3382032424 0 0) 0x7f2342ef4400 con 0x7f2342dd4800
  -132> 2017-11-05 21:59:34.380507 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342930900 for mon.0 10.10.1.21:6789/0
  -131> 2017-11-05 21:59:34.380516 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
  -130> 2017-11-05 21:59:34.380519 7f23302e4700 20 is_capable service=mon command= read on cap allow *
  -129> 2017-11-05 21:59:34.380522 7f23302e4700 20  allow so far , doing grant allow *
  -128> 2017-11-05 21:59:34.380523 7f23302e4700 20  allow all
  -127> 2017-11-05 21:59:34.380524 7f23302e4700 20 is_capable service=mon command= exec on cap allow *
  -126> 2017-11-05 21:59:34.380526 7f23302e4700 20  allow so far , doing grant allow *
  -125> 2017-11-05 21:59:34.380527 7f23302e4700 20  allow all
  -124> 2017-11-05 21:59:34.380532 7f23302e4700 10 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907758) handle_lease on 112907758 now 2017-11-05 21:59:39.341409
  -123> 2017-11-05 21:59:34.380547 7f23302e4700  1 -- 10.10.1.23:6789/0 --> 10.10.1.21:6789/0 -- paxos(lease_ack lc 112907758 fc 112907080 pn 0 opn 0) v4 -- 0x7f2342ef2600 con 0
  -122> 2017-11-05 21:59:34.380564 7f23302e4700 20 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907758) reset_lease_timeout - setting timeout event
  -121> 2017-11-05 21:59:34.380572 7f23302e4700  5 mon.ceph03@2(peon).monmap v14 apply_mon_features wait for service to be writeable
  -120> 2017-11-05 21:59:34.380576 7f23302e4700  5 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907758) is_readable = 1 - now=2017-11-05 21:59:34.380576 lease_expire=2017-11-05 21:59:39.341409 has v0 lc 112907758
  -119> 2017-11-05 21:59:34.380616 7f232c2dc700 10 _calc_signature seq 51 front_crc_ = 221321949 middle_crc = 0 data_crc = 0 sig = 13265552358249501418
  -118> 2017-11-05 21:59:34.380633 7f232c2dc700 20 Putting signature in client message(seq # 51): sig = 13265552358249501418
  -117> 2017-11-05 21:59:36.016471 7f232d2de700  1 -- 10.10.1.23:6789/0 >> - conn(0x7f2342f06000 :6789 s=STATE_ACCEPTING pgs=0 cs=0 l=0)._process_connection sd=37 -
  -116> 2017-11-05 21:59:36.016618 7f232d2de700 10 mon.ceph03@2(peon) e14 ms_verify_authorizer 10.10.1.23:0/3191937035 client protocol 0
  -115> 2017-11-05 21:59:36.016641 7f232d2de700 10 In get_auth_session_handler for protocol 0
  -114> 2017-11-05 21:59:36.016674 7f232d2de700  2 -- 10.10.1.23:6789/0 >> 10.10.1.23:0/3191937035 conn(0x7f2342f06000 :6789 s=STATE_ACCEPTING_WAIT_SEQ pgs=2 cs=1 l=1).handle_connect_msg accept write reply msg done
  -113> 2017-11-05 21:59:36.016711 7f232d2de700  2 -- 10.10.1.23:6789/0 >> 10.10.1.23:0/3191937035 conn(0x7f2342f06000 :6789 s=STATE_ACCEPTING_WAIT_SEQ pgs=2 cs=1 l=1)._process_connection accept get newly_acked_seq 0
  -112> 2017-11-05 21:59:36.016785 7f232d2de700  5 -- 10.10.1.23:6789/0 >> 10.10.1.23:0/3191937035 conn(0x7f2342f06000 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=2 cs=1 l=1). rx client.? seq 1 0x7f234296d900 auth(proto 0 30 bytes epoch 0) v1
  -111> 2017-11-05 21:59:36.016822 7f23302e4700  1 -- 10.10.1.23:6789/0 <== client.? 10.10.1.23:0/3191937035 1 ==== auth(proto 0 30 bytes epoch 0) v1 ==== 60+0+0 (673663173 0 0) 0x7f234296d900 con 0x7f2342f06000
  -110> 2017-11-05 21:59:36.016852 7f23302e4700 10 mon.ceph03@2(peon) e14 _ms_dispatch new session 0x7f2342933600 MonSession(client.? 10.10.1.23:0/3191937035 is open ) features 0x1ffddff8eea4fffb
  -109> 2017-11-05 21:59:36.016864 7f23302e4700 20 mon.ceph03@2(peon) e14  caps
  -108> 2017-11-05 21:59:36.016867 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(auth 26252..26332) dispatch 0x7f234296d900 auth(proto 0 30 bytes epoch 0) v1 from client.? 10.10.1.23:0/3191937035 con 0x7f2342f06000
  -107> 2017-11-05 21:59:36.016874 7f23302e4700  5 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907758) is_readable = 1 - now=2017-11-05 21:59:36.016874 lease_expire=2017-11-05 21:59:39.341409 has v0 lc 112907758
  -106> 2017-11-05 21:59:36.016884 7f23302e4700 10 mon.ceph03@2(peon).auth v26332 preprocess_query auth(proto 0 30 bytes epoch 0) v1 from client.? 10.10.1.23:0/3191937035
  -105> 2017-11-05 21:59:36.016889 7f23302e4700 10 mon.ceph03@2(peon).auth v26332 prep_auth() blob_size=30
  -104> 2017-11-05 21:59:36.016900 7f23302e4700 10 mon.ceph03@2(peon).auth v26332 AuthMonitor::assign_global_id m=auth(proto 0 30 bytes epoch 0) v1 mon=2/3 last_allocated=55234103 max_global_id=55244096
  -103> 2017-11-05 21:59:36.016904 7f23302e4700 10 mon.ceph03@2(peon).auth v26332 next_global_id should be 55234106
  -102> 2017-11-05 21:59:36.016920 7f23302e4700  1 -- 10.10.1.23:6789/0 --> 10.10.1.23:0/3191937035 -- mon_map magic: 0 v1 -- 0x7f234298f840 con 0
  -101> 2017-11-05 21:59:36.016951 7f23302e4700 10 cephx server client.admin: start_session server_challenge 7f99fad6b5c5d101
  -100> 2017-11-05 21:59:36.016960 7f23302e4700  2 mon.ceph03@2(peon) e14 send_reply 0x7f2342e9d4e0 0x7f2342f14500 auth_reply(proto 2 0 (0) Success) v1
   -99> 2017-11-05 21:59:36.016985 7f23302e4700  1 -- 10.10.1.23:6789/0 --> 10.10.1.23:0/3191937035 -- auth_reply(proto 2 0 (0) Success) v1 -- 0x7f2342f14500 con 0
   -98> 2017-11-05 21:59:36.017375 7f232d2de700  5 -- 10.10.1.23:6789/0 >> 10.10.1.23:0/3191937035 conn(0x7f2342f06000 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=2 cs=1 l=1). rx client.? seq 2 0x7f2342f14500 auth(proto 2 32 bytes epoch 0) v1
   -97> 2017-11-05 21:59:36.017407 7f23302e4700  1 -- 10.10.1.23:6789/0 <== client.? 10.10.1.23:0/3191937035 2 ==== auth(proto 2 32 bytes epoch 0) v1 ==== 62+0+0 (3448023605 0 0) 0x7f2342f14500 con 0x7f2342f06000
   -96> 2017-11-05 21:59:36.017428 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342933600 for client.? 10.10.1.23:0/3191937035
   -95> 2017-11-05 21:59:36.017436 7f23302e4700 20 mon.ceph03@2(peon) e14  caps
   -94> 2017-11-05 21:59:36.017438 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(auth 26252..26332) dispatch 0x7f2342f14500 auth(proto 2 32 bytes epoch 0) v1 from client.? 10.10.1.23:0/3191937035 con 0x7f2342f06000
   -93> 2017-11-05 21:59:36.017443 7f23302e4700  5 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907758) is_readable = 1 - now=2017-11-05 21:59:36.017444 lease_expire=2017-11-05 21:59:39.341409 has v0 lc 112907758
   -92> 2017-11-05 21:59:36.017450 7f23302e4700 10 mon.ceph03@2(peon).auth v26332 preprocess_query auth(proto 2 32 bytes epoch 0) v1 from client.? 10.10.1.23:0/3191937035
   -91> 2017-11-05 21:59:36.017456 7f23302e4700 10 mon.ceph03@2(peon).auth v26332 prep_auth() blob_size=32
   -90> 2017-11-05 21:59:36.017459 7f23302e4700 10 cephx server client.admin: handle_request get_auth_session_key for client.admin
   -89> 2017-11-05 21:59:36.017492 7f23302e4700 20 cephx server client.admin:  checking key: req.key=df499f5ee9b702d1 expected_key=df499f5ee9b702d1
   -88> 2017-11-05 21:59:36.017542 7f23302e4700 10 cephx: build_service_ticket_reply encoding 1 tickets with secret AQAh1Z5VjAN3ARAAw+R6HvdEfcFucLX1YSmG4Q==
   -87> 2017-11-05 21:59:36.017565 7f23302e4700 10 cephx: build_service_ticket service auth secret_id 1699 ticket_info.ticket.name=client.admin
   -86> 2017-11-05 21:59:36.017579 7f23302e4700 10 cephx keyserverdata: get_caps: name=client.admin
   -85> 2017-11-05 21:59:36.017583 7f23302e4700 10 cephx keyserverdata: get_secret: num of caps=4
   -84> 2017-11-05 21:59:36.017629 7f23302e4700  2 mon.ceph03@2(peon) e14 send_reply 0x7f2342e9d4e0 0x7f234296d900 auth_reply(proto 2 0 (0) Success) v1
   -83> 2017-11-05 21:59:36.017642 7f23302e4700  1 -- 10.10.1.23:6789/0 --> 10.10.1.23:0/3191937035 -- auth_reply(proto 2 0 (0) Success) v1 -- 0x7f234296d900 con 0
   -82> 2017-11-05 21:59:36.017945 7f232d2de700  5 -- 10.10.1.23:6789/0 >> 10.10.1.23:0/3191937035 conn(0x7f2342f06000 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=2 cs=1 l=1). rx client.? seq 3 0x7f234296d900 auth(proto 2 165 bytes epoch 0) v1
   -81> 2017-11-05 21:59:36.017978 7f23302e4700  1 -- 10.10.1.23:6789/0 <== client.? 10.10.1.23:0/3191937035 3 ==== auth(proto 2 165 bytes epoch 0) v1 ==== 195+0+0 (3388592148 0 0) 0x7f234296d900 con 0x7f2342f06000
   -80> 2017-11-05 21:59:36.017999 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342933600 for client.? 10.10.1.23:0/3191937035
   -79> 2017-11-05 21:59:36.018007 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
   -78> 2017-11-05 21:59:36.018009 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(auth 26252..26332) dispatch 0x7f234296d900 auth(proto 2 165 bytes epoch 0) v1 from client.? 10.10.1.23:0/3191937035 con 0x7f2342f06000
   -77> 2017-11-05 21:59:36.018014 7f23302e4700  5 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907758) is_readable = 1 - now=2017-11-05 21:59:36.018014 lease_expire=2017-11-05 21:59:39.341409 has v0 lc 112907758
   -76> 2017-11-05 21:59:36.018021 7f23302e4700 10 mon.ceph03@2(peon).auth v26332 preprocess_query auth(proto 2 165 bytes epoch 0) v1 from client.? 10.10.1.23:0/3191937035
   -75> 2017-11-05 21:59:36.018025 7f23302e4700 10 mon.ceph03@2(peon).auth v26332 prep_auth() blob_size=165
   -74> 2017-11-05 21:59:36.018027 7f23302e4700 10 cephx server client.admin: handle_request get_principal_session_key
   -73> 2017-11-05 21:59:36.018030 7f23302e4700 10 cephx: verify_authorizer decrypted service auth secret_id=1699
   -72> 2017-11-05 21:59:36.018071 7f23302e4700 10 cephx: verify_authorizer global_id=55234106
   -71> 2017-11-05 21:59:36.018091 7f23302e4700 10 cephx: verify_authorizer ok nonce 6b8b4567327b23c6 reply_bl.length()=36
   -70> 2017-11-05 21:59:36.018096 7f23302e4700 10 cephx server client.admin:  ticket_req.keys = 21
   -69> 2017-11-05 21:59:36.018097 7f23302e4700 10 cephx server client.admin:  adding key for service mon
   -68> 2017-11-05 21:59:36.018134 7f23302e4700 10 cephx server client.admin:  adding key for service osd
   -67> 2017-11-05 21:59:36.018160 7f23302e4700 10 cephx keyserverdata: get_caps: name=client.admin
   -66> 2017-11-05 21:59:36.018166 7f23302e4700 10 cephx keyserverdata: get_secret: num of caps=4
   -65> 2017-11-05 21:59:36.018169 7f23302e4700 10 cephx server client.admin:  adding key for service mgr
   -64> 2017-11-05 21:59:36.018191 7f23302e4700 10 cephx keyserverdata: get_caps: name=client.admin
   -63> 2017-11-05 21:59:36.018196 7f23302e4700 10 cephx keyserverdata: get_secret: num of caps=4
   -62> 2017-11-05 21:59:36.018201 7f23302e4700 10 cephx: build_service_ticket_reply encoding 3 tickets with secret AQC4e/9ZZUwLARAA4eu3mAYpVCYcqDn+3vicJQ==
   -61> 2017-11-05 21:59:36.018218 7f23302e4700 10 cephx: build_service_ticket service mon secret_id 20359 ticket_info.ticket.name=client.admin
   -60> 2017-11-05 21:59:36.018240 7f23302e4700 10 cephx: build_service_ticket service osd secret_id 20359 ticket_info.ticket.name=client.admin
   -59> 2017-11-05 21:59:36.018263 7f23302e4700 10 cephx: build_service_ticket service mgr secret_id 7 ticket_info.ticket.name=client.admin
   -58> 2017-11-05 21:59:36.018291 7f23302e4700  2 mon.ceph03@2(peon) e14 send_reply 0x7f2342e9d4e0 0x7f2342f14500 auth_reply(proto 2 0 (0) Success) v1
   -57> 2017-11-05 21:59:36.018303 7f23302e4700  1 -- 10.10.1.23:6789/0 --> 10.10.1.23:0/3191937035 -- auth_reply(proto 2 0 (0) Success) v1 -- 0x7f2342f14500 con 0
   -56> 2017-11-05 21:59:36.018712 7f232d2de700  5 -- 10.10.1.23:6789/0 >> 10.10.1.23:0/3191937035 conn(0x7f2342f06000 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=2 cs=1 l=1). rx client.? seq 4 0x7f234298f840 mon_subscribe({monmap=0+}) v2
   -55> 2017-11-05 21:59:36.018759 7f23302e4700  1 -- 10.10.1.23:6789/0 <== client.? 10.10.1.23:0/3191937035 4 ==== mon_subscribe({monmap=0+}) v2 ==== 23+0+0 (1620593354 0 0) 0x7f234298f840 con 0x7f2342f06000
   -54> 2017-11-05 21:59:36.018777 7f232d2de700  5 -- 10.10.1.23:6789/0 >> 10.10.1.23:0/3191937035 conn(0x7f2342f06000 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=2 cs=1 l=1). rx client.55234106 seq 5 0x7f234298fa80 mon_subscribe({mgrmap=0+}) v2
   -53> 2017-11-05 21:59:36.018786 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342933600 for client.? 10.10.1.23:0/3191937035
   -52> 2017-11-05 21:59:36.018804 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
   -51> 2017-11-05 21:59:36.018808 7f23302e4700 20 is_capable service=mon command= read on cap allow *
   -50> 2017-11-05 21:59:36.018813 7f23302e4700 20  allow so far , doing grant allow *
   -49> 2017-11-05 21:59:36.018814 7f23302e4700 20  allow all
   -48> 2017-11-05 21:59:36.018818 7f23302e4700 10 mon.ceph03@2(peon) e14 handle_subscribe mon_subscribe({monmap=0+}) v2
   -47> 2017-11-05 21:59:36.018832 7f23302e4700 10 mon.ceph03@2(peon).monmap v14 check_sub monmap next 0 have 14
   -46> 2017-11-05 21:59:36.018847 7f23302e4700  1 -- 10.10.1.23:6789/0 --> 10.10.1.23:0/3191937035 -- mon_map magic: 0 v1 -- 0x7f2342f50d80 con 0
   -45> 2017-11-05 21:59:36.018868 7f23302e4700  1 -- 10.10.1.23:6789/0 <== client.55234106 10.10.1.23:0/3191937035 5 ==== mon_subscribe({mgrmap=0+}) v2 ==== 23+0+0 (2287485974 0 0) 0x7f234298fa80 con 0x7f2342f06000
   -44> 2017-11-05 21:59:36.018872 7f232d2de700  5 -- 10.10.1.23:6789/0 >> 10.10.1.23:0/3191937035 conn(0x7f2342f06000 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=2 cs=1 l=1). rx client.55234106 seq 6 0x7f234298fcc0 mon_subscribe({osdmap=0}) v2
   -43> 2017-11-05 21:59:36.018880 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342933600 for client.? 10.10.1.23:0/3191937035
   -42> 2017-11-05 21:59:36.018885 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
   -41> 2017-11-05 21:59:36.018889 7f23302e4700 20 is_capable service=mon command= read on cap allow *
   -40> 2017-11-05 21:59:36.018891 7f23302e4700 20  allow so far , doing grant allow *
   -39> 2017-11-05 21:59:36.018892 7f23302e4700 20  allow all
   -38> 2017-11-05 21:59:36.018893 7f23302e4700 10 mon.ceph03@2(peon) e14 handle_subscribe mon_subscribe({mgrmap=0+}) v2
   -37> 2017-11-05 21:59:36.018899 7f23302e4700 20 mon.ceph03@2(peon).mgr e394 Sending map to subscriber 0x7f2342f06000 10.10.1.23:0/3191937035
   -36> 2017-11-05 21:59:36.018907 7f23302e4700  1 -- 10.10.1.23:6789/0 --> 10.10.1.23:0/3191937035 -- mgrmap(e 394) v1 -- 0x7f2342bebb80 con 0
   -35> 2017-11-05 21:59:36.018919 7f23302e4700  1 -- 10.10.1.23:6789/0 <== client.55234106 10.10.1.23:0/3191937035 6 ==== mon_subscribe({osdmap=0}) v2 ==== 23+0+0 (319581110 0 0) 0x7f234298fcc0 con 0x7f2342f06000
   -34> 2017-11-05 21:59:36.018926 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342933600 for client.? 10.10.1.23:0/3191937035
   -33> 2017-11-05 21:59:36.018929 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
   -32> 2017-11-05 21:59:36.018931 7f23302e4700 20 is_capable service=mon command= read on cap allow *
   -31> 2017-11-05 21:59:36.018933 7f23302e4700 20  allow so far , doing grant allow *
   -30> 2017-11-05 21:59:36.018934 7f23302e4700 20  allow all
   -29> 2017-11-05 21:59:36.018935 7f23302e4700 10 mon.ceph03@2(peon) e14 handle_subscribe mon_subscribe({osdmap=0}) v2
   -28> 2017-11-05 21:59:36.018938 7f23302e4700 20 is_capable service=osd command= read on cap allow *
   -27> 2017-11-05 21:59:36.018940 7f23302e4700 20  allow so far , doing grant allow *
   -26> 2017-11-05 21:59:36.018941 7f23302e4700 20  allow all
   -25> 2017-11-05 21:59:36.018942 7f23302e4700 10 mon.ceph03@2(peon).osd e1105 check_osdmap_sub 0x7f2342dc2fc0 next 0 (onetime)
   -24> 2017-11-05 21:59:36.018965 7f23302e4700  1 -- 10.10.1.23:6789/0 --> 10.10.1.23:0/3191937035 -- osd_map(1105..1105 src has 503..1105) v3 -- 0x7f234296d900 con 0
   -23> 2017-11-05 21:59:36.024167 7f232d2de700  5 -- 10.10.1.23:6789/0 >> 10.10.1.23:0/3191937035 conn(0x7f2342f06000 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=2 cs=1 l=1). rx client.55234106 seq 7 0x7f2342f50d80 mon_command({"prefix": "get_command_descriptions"} v 0) v1
   -22> 2017-11-05 21:59:36.024204 7f23302e4700  1 -- 10.10.1.23:6789/0 <== client.55234106 10.10.1.23:0/3191937035 7 ==== mon_command({"prefix": "get_command_descriptions"} v 0) v1 ==== 80+0+0 (1793402910 0 0) 0x7f2342f50d80 con 0x7f2342f06000
   -21> 2017-11-05 21:59:36.024233 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342933600 for client.? 10.10.1.23:0/3191937035
   -20> 2017-11-05 21:59:36.024242 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
   -19> 2017-11-05 21:59:36.028374 7f23302e4700  2 mon.ceph03@2(peon) e14 send_reply 0x7f2342f11f80 0x7f2342f50900 mon_command_ack([{"prefix": "get_command_descriptions"}]=0  v0) v1
   -18> 2017-11-05 21:59:36.028385 7f23302e4700  1 -- 10.10.1.23:6789/0 --> 10.10.1.23:0/3191937035 -- mon_command_ack([{"prefix": "get_command_descriptions"}]=0  v0) v1 -- 0x7f2342f50900 con 0
   -17> 2017-11-05 21:59:36.167349 7f232d2de700  5 -- 10.10.1.23:6789/0 >> 10.10.1.23:0/3191937035 conn(0x7f2342f06000 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=2 cs=1 l=1). rx client.55234106 seq 8 0x7f2342f50900 mon_command({"prefix": "osd pool stats"} v 0) v1
   -16> 2017-11-05 21:59:36.167394 7f23302e4700  1 -- 10.10.1.23:6789/0 <== client.55234106 10.10.1.23:0/3191937035 8 ==== mon_command({"prefix": "osd pool stats"} v 0) v1 ==== 70+0+0 (222854354 0 0) 0x7f2342f50900 con 0x7f2342f06000
   -15> 2017-11-05 21:59:36.167424 7f23302e4700 20 mon.ceph03@2(peon) e14 _ms_dispatch existing session 0x7f2342933600 for client.? 10.10.1.23:0/3191937035
   -14> 2017-11-05 21:59:36.167435 7f23302e4700 20 mon.ceph03@2(peon) e14  caps allow *
   -13> 2017-11-05 21:59:36.167472 7f23302e4700  0 mon.ceph03@2(peon) e14 handle_command mon_command({"prefix": "osd pool stats"} v 0) v1
   -12> 2017-11-05 21:59:36.167521 7f23302e4700 20 is_capable service=osd command=osd pool stats read on cap allow *
   -11> 2017-11-05 21:59:36.167528 7f23302e4700 20  allow so far , doing grant allow *
   -10> 2017-11-05 21:59:36.167530 7f23302e4700 20  allow all
    -9> 2017-11-05 21:59:36.167531 7f23302e4700 10 mon.ceph03@2(peon) e14 _allowed_command capable
    -8> 2017-11-05 21:59:36.167544 7f23302e4700  0 log_channel(audit) log [DBG] : from='client.? 10.10.1.23:0/3191937035' entity='client.admin' cmd=[{"prefix": "osd pool stats"}]: dispatch
    -7> 2017-11-05 21:59:36.167547 7f23302e4700 10 log_client _send_to_monlog to self
    -6> 2017-11-05 21:59:36.167549 7f23302e4700 10 log_client  log_queue is 1 last_log 2 sent 1 num 1 unsent 1 sending 1
    -5> 2017-11-05 21:59:36.167553 7f23302e4700 10 log_client  will send 2017-11-05 21:59:36.167546 mon.ceph03 mon.2 10.10.1.23:6789/0 2 : audit [DBG] from='client.? 10.10.1.23:0/3191937035' entity='client.admin' cmd=[{"prefix": "osd pool stats"}]: dispatch
    -4> 2017-11-05 21:59:36.167567 7f23302e4700  1 -- 10.10.1.23:6789/0 --> 10.10.1.23:6789/0 -- log(1 entries from seq 2 at 2017-11-05 21:59:36.167546) v1 -- 0x7f234296f700 con 0
    -3> 2017-11-05 21:59:36.167585 7f23302e4700 10 mon.ceph03@2(peon).paxosservice(osdmap 503..1105) dispatch 0x7f2342f50900 mon_command({"prefix": "osd pool stats"} v 0) v1 from client.55234106 10.10.1.23:0/3191937035 con 0x7f2342f06000
    -2> 2017-11-05 21:59:36.167596 7f23302e4700  5 mon.ceph03@2(peon).paxos(paxos active c 112907080..112907758) is_readable = 1 - now=2017-11-05 21:59:36.167596 lease_expire=2017-11-05 21:59:39.341409 has v0 lc 112907758
    -1> 2017-11-05 21:59:36.167618 7f23302e4700 10 mon.ceph03@2(peon).osd e1105 preprocess_query mon_command({"prefix": "osd pool stats"} v 0) v1 from client.55234106 10.10.1.23:0/3191937035
     0> 2017-11-05 21:59:36.173253 7f23302e4700 -1 *** Caught signal (Aborted) **
 in thread 7f23302e4700 thread_name:ms_dispatch

 ceph version 12.2.1 (3e7492b9ada8bdc9a5cd0feafd42fbca27f9c38e) luminous (stable)
 1: (()+0x8c5429) [0x7f233948c429]
 2: (()+0x10330) [0x7f2337f5f330]
 3: (gsignal()+0x37) [0x7f2336750c37]
 4: (abort()+0x148) [0x7f2336754028]
 5: (()+0x447c19) [0x7f233900ec19]
 6: (OSDMonitor::preprocess_command(boost::intrusive_ptr<MonOpRequest>)+0xd5e) [0x7f23390de96e]
 7: (OSDMonitor::preprocess_query(boost::intrusive_ptr<MonOpRequest>)+0x3d6) [0x7f23390e7846]
 8: (PaxosService::dispatch(boost::intrusive_ptr<MonOpRequest>)+0x841) [0x7f23390989b1]
 9: (Monitor::handle_command(boost::intrusive_ptr<MonOpRequest>)+0x1c69) [0x7f2338f7f069]
 10: (Monitor::dispatch_op(boost::intrusive_ptr<MonOpRequest>)+0x91f) [0x7f2338f849bf]
 11: (Monitor::_ms_dispatch(Message*)+0x7b9) [0x7f2338f85cb9]
 12: (Monitor::ms_dispatch(Message*)+0x23) [0x7f2338faefb3]
 13: (DispatchQueue::entry()+0x78b) [0x7f233943921b]
 14: (DispatchQueue::DispatchThread::entry()+0xd) [0x7f233924162d]
 15: (()+0x8184) [0x7f2337f57184]
 16: (clone()+0x6d) [0x7f2336817ffd]
 NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.

--- logging levels ---
   0/ 5 none
   0/ 1 lockdep
   0/ 1 context
   1/ 1 crush
   1/ 5 mds
   1/ 5 mds_balancer
   1/ 5 mds_locker
   1/ 5 mds_log
   1/ 5 mds_log_expire
   1/ 5 mds_migrator
   0/ 1 buffer
   0/ 1 timer
   0/ 1 filer
   0/ 1 striper
   0/ 1 objecter
   0/ 5 rados
   0/ 5 rbd
   0/ 5 rbd_mirror
   0/ 5 rbd_replay
   0/ 5 journaler
   0/ 5 objectcacher
   0/ 5 client
   1/ 5 osd
   0/ 5 optracker
   0/ 5 objclass
   1/ 3 filestore
   1/ 3 journal
   0/ 5 ms
  20/20 mon
   0/10 monc
  20/20 paxos
   0/ 5 tp
  20/20 auth
   1/ 5 crypto
   1/ 1 finisher
   1/ 5 heartbeatmap
   1/ 5 perfcounter
   1/ 5 rgw
   1/10 civetweb
   1/ 5 javaclient
   1/ 5 asok
   1/ 1 throttle
   0/ 0 refs
   1/ 5 xio
   1/ 5 compressor
   1/ 5 bluestore
   1/ 5 bluefs
   1/ 3 bdev
   1/ 5 kstore
   4/ 5 rocksdb
   4/ 5 leveldb
   4/ 5 memdb
   1/ 5 kinetic
   1/ 5 fuse
   1/ 5 mgr
   1/ 5 mgrc
   1/ 5 dpdk
   1/ 5 eventtrace
  -2/-2 (syslog threshold)
  -1/-1 (stderr threshold)
  max_recent     10000
  max_new         1000
  log_file /var/log/ceph/ceph-mon.ceph03.log
--- end dump of recent events ---