-- Logs begin at Thu 2022-08-18 20:42:50 UTC, end at Fri 2022-08-19 02:14:25 UTC. -- Aug 18 21:38:13.485715 n-d-853704-1 systemd[1]: Started Devstack devstack@n-super-cond.service. Aug 18 21:38:14.151329 n-d-853704-1 nova-conductor[119706]: Modules with known eventlet monkey patching issues were imported prior to eventlet monkey patching: urllib3. This warning can usually be ignored if the caller is only importing and not executing nova code. Aug 18 21:38:17.277199 n-d-853704-1 nova-conductor[119706]: INFO dbcounter [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] Registered counter for database nova_api Aug 18 21:38:17.295417 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_db.sqlalchemy.engines [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] MySQL server mode set to STRICT_TRANS_TABLES,STRICT_ALL_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,TRADITIONAL,NO_ENGINE_SUBSTITUTION {{(pid=119706) _check_effective_sql_mode /usr/local/lib/python3.8/dist-packages/oslo_db/sqlalchemy/engines.py:314}} Aug 18 21:38:17.301809 n-d-853704-1 nova-conductor[119706]: DEBUG dbcounter [-] [119706] Writer thread running {{(pid=119706) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:99}} Aug 18 21:38:17.355149 n-d-853704-1 nova-conductor[119706]: DEBUG nova.context [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] Found 2 cells: 00000000-0000-0000-0000-000000000000(cell0),33319d98-cfe4-4ac2-b757-392710e3e602(cell1) {{(pid=119706) load_cells /opt/stack/nova/nova/context.py:464}} Aug 18 21:38:17.359264 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_concurrency.lockutils [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=119706) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 21:38:17.359887 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_concurrency.lockutils [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=119706) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 21:38:17.361433 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_concurrency.lockutils [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.002s {{(pid=119706) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 21:38:17.362225 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_concurrency.lockutils [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=119706) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 21:38:17.362747 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_concurrency.lockutils [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=119706) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 21:38:17.364853 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_concurrency.lockutils [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.002s {{(pid=119706) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 21:38:17.366758 n-d-853704-1 nova-conductor[119706]: INFO dbcounter [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] Registered counter for database nova_cell0 Aug 18 21:38:17.370471 n-d-853704-1 nova-conductor[119706]: INFO dbcounter [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] Registered counter for database nova_cell1 Aug 18 21:38:17.380093 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_db.sqlalchemy.engines [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] MySQL server mode set to STRICT_TRANS_TABLES,STRICT_ALL_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,TRADITIONAL,NO_ENGINE_SUBSTITUTION {{(pid=119706) _check_effective_sql_mode /usr/local/lib/python3.8/dist-packages/oslo_db/sqlalchemy/engines.py:314}} Aug 18 21:38:17.385541 n-d-853704-1 nova-conductor[119706]: DEBUG dbcounter [-] [119706] Writer thread running {{(pid=119706) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:99}} Aug 18 21:38:17.386245 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_db.sqlalchemy.engines [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] MySQL server mode set to STRICT_TRANS_TABLES,STRICT_ALL_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,TRADITIONAL,NO_ENGINE_SUBSTITUTION {{(pid=119706) _check_effective_sql_mode /usr/local/lib/python3.8/dist-packages/oslo_db/sqlalchemy/engines.py:314}} Aug 18 21:38:17.519045 n-d-853704-1 nova-conductor[119706]: DEBUG dbcounter [-] [119706] Writer thread running {{(pid=119706) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:99}} Aug 18 21:38:17.526896 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_concurrency.lockutils [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] Acquiring lock "singleton_lock" {{(pid=119706) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 18 21:38:17.527384 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_concurrency.lockutils [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] Acquired lock "singleton_lock" {{(pid=119706) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 18 21:38:17.528251 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_concurrency.lockutils [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] Releasing lock "singleton_lock" {{(pid=119706) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 18 21:38:17.529079 n-d-853704-1 nova-conductor[119706]: INFO oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] Starting 2 workers Aug 18 21:38:17.534408 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] Started child 120657 {{(pid=119706) _start_child /usr/local/lib/python3.8/dist-packages/oslo_service/service.py:575}} Aug 18 21:38:17.539396 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] Started child 120658 {{(pid=119706) _start_child /usr/local/lib/python3.8/dist-packages/oslo_service/service.py:575}} Aug 18 21:38:17.540892 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] Full set of CONF: {{(pid=119706) wait /usr/local/lib/python3.8/dist-packages/oslo_service/service.py:649}} Aug 18 21:38:17.541499 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] ******************************************************************************** {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2589}} Aug 18 21:38:17.542060 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] Configuration options gathered from: {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2590}} Aug 18 21:38:17.542685 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] command line args: ['--config-file', '/etc/nova/nova.conf'] {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2591}} Aug 18 21:38:17.544163 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] config files: ['/etc/nova/nova.conf'] {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2592}} Aug 18 21:38:17.544731 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] ================================================================================ {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2594}} Aug 18 21:38:17.545331 n-d-853704-1 nova-conductor[120657]: INFO nova.service [-] Starting conductor node (version 25.1.0) Aug 18 21:38:17.546331 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] allow_resize_to_same_host = True {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.547043 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] arq_binding_timeout = 300 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.547733 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] block_device_allocate_retries = 60 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.548386 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] block_device_allocate_retries_interval = 3 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.549102 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] cert = self.pem {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.549751 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] compute_driver = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.550470 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] compute_monitors = [] {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.551201 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] config_dir = [] {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.551986 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] config_drive_format = iso9660 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.552569 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] config_file = ['/etc/nova/nova.conf'] {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.553463 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] config_source = [] {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.554121 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] console_host = n-d-853704-1 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.554791 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] control_exchange = nova {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.555009 n-d-853704-1 nova-conductor[120657]: INFO dbcounter [None req-e877d9a8-54e5-4cd5-b86d-e17960e5fb33 None None] Registered counter for database nova_cell0 Aug 18 21:38:17.555685 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] cpu_allocation_ratio = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.556320 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] daemon = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.556980 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] debug = True {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.557609 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] default_access_ip_network_name = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.558273 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] default_availability_zone = nova {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.559458 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] default_ephemeral_format = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.559857 n-d-853704-1 nova-conductor[120658]: INFO nova.service [-] Starting conductor node (version 25.1.0) Aug 18 21:38:17.561112 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] default_log_levels = ['amqp=WARN', 'amqplib=WARN', 'boto=WARN', 'qpid=WARN', 'sqlalchemy=WARN', 'suds=INFO', 'oslo.messaging=INFO', 'oslo_messaging=INFO', 'iso8601=WARN', 'requests.packages.urllib3.connectionpool=WARN', 'urllib3.connectionpool=WARN', 'websocket=WARN', 'requests.packages.urllib3.util.retry=WARN', 'urllib3.util.retry=WARN', 'keystonemiddleware=WARN', 'routes.middleware=WARN', 'stevedore=WARN', 'taskflow=WARN', 'keystoneauth=WARN', 'oslo.cache=INFO', 'oslo_policy=INFO', 'dogpile.core.dogpile=INFO', 'glanceclient=WARN', 'oslo.privsep.daemon=INFO'] {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.561796 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] default_schedule_zone = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.562673 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] disk_allocation_ratio = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.564020 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] enable_new_services = True {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.565083 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] enabled_apis = ['osapi_compute'] {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.566048 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] enabled_ssl_apis = [] {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.567002 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] flat_injected = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.567928 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] force_config_drive = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.568731 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] force_raw_images = True {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.569403 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] graceful_shutdown_timeout = 5 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.570061 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] heal_instance_info_cache_interval = 60 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.571180 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] host = n-d-853704-1 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.571288 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_db.sqlalchemy.engines [None req-e877d9a8-54e5-4cd5-b86d-e17960e5fb33 None None] MySQL server mode set to STRICT_TRANS_TABLES,STRICT_ALL_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,TRADITIONAL,NO_ENGINE_SUBSTITUTION {{(pid=120657) _check_effective_sql_mode /usr/local/lib/python3.8/dist-packages/oslo_db/sqlalchemy/engines.py:314}} Aug 18 21:38:17.571990 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] initial_cpu_allocation_ratio = 16.0 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.572633 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] initial_disk_allocation_ratio = 1.0 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.573272 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] initial_ram_allocation_ratio = 1.5 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.574156 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] injected_network_template = /opt/stack/nova/nova/virt/interfaces.template {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.574854 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] instance_build_timeout = 0 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.575484 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] instance_delete_interval = 300 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.576153 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] instance_format = [instance: %(uuid)s] {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.576774 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] instance_name_template = instance-%08x {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.577405 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] instance_usage_audit = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.578061 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] instance_usage_audit_period = month {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.578727 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] instance_uuid_format = [instance: %(uuid)s] {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.579038 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writer thread running {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:99}} Aug 18 21:38:17.579616 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] instances_path = /opt/stack/data/nova/instances {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.580297 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] internal_service_availability_zone = internal {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.580913 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] key = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.581021 n-d-853704-1 nova-conductor[120658]: INFO dbcounter [None req-941b5ded-1cec-4e4b-8ef3-7538d164155f None None] Registered counter for database nova_cell0 Aug 18 21:38:17.581674 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] live_migration_retry_count = 30 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.582320 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] log_config_append = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.582989 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] log_date_format = %Y-%m-%d %H:%M:%S {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.583615 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] log_dir = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.584266 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] log_file = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.584842 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] log_options = True {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.585455 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] log_rotate_interval = 1 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.586436 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] log_rotate_interval_type = days {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.587070 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] log_rotation_type = none {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.587642 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] logging_context_format_string = %(color)s%(levelname)s %(name)s [%(global_request_id)s %(request_id)s %(project_name)s %(user_name)s%(color)s] %(instance)s%(color)s%(message)s {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.588229 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] logging_debug_format_suffix = {{(pid=%(process)d) %(funcName)s %(pathname)s:%(lineno)d}} {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.588878 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] logging_default_format_string = %(color)s%(levelname)s %(name)s [-%(color)s] %(instance)s%(color)s%(message)s {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.589537 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] logging_exception_prefix = ERROR %(name)s %(instance)s {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.590198 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] logging_user_identity_format = %(user)s %(project)s %(domain)s %(system_scope)s %(user_domain)s %(project_domain)s {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.590953 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] long_rpc_timeout = 1800 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.591561 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] max_concurrent_builds = 10 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.592237 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] max_concurrent_live_migrations = 1 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.592841 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] max_concurrent_snapshots = 5 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.593464 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] max_local_block_devices = 3 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.594077 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] max_logfile_count = 30 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.594718 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] max_logfile_size_mb = 200 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.595359 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] maximum_instance_delete_attempts = 5 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.596016 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] metadata_listen = 0.0.0.0 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.597210 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] metadata_listen_port = 8775 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.601803 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] metadata_workers = 2 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.602338 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_db.sqlalchemy.engines [None req-941b5ded-1cec-4e4b-8ef3-7538d164155f None None] MySQL server mode set to STRICT_TRANS_TABLES,STRICT_ALL_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,TRADITIONAL,NO_ENGINE_SUBSTITUTION {{(pid=120658) _check_effective_sql_mode /usr/local/lib/python3.8/dist-packages/oslo_db/sqlalchemy/engines.py:314}} Aug 18 21:38:17.602777 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] migrate_max_retries = -1 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.603452 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] mkisofs_cmd = genisoimage {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.604236 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] my_block_storage_ip = 10.222.0.49 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.604801 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] my_ip = 10.222.0.49 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.605476 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] network_allocate_retries = 0 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.606654 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] non_inheritable_image_properties = ['cache_in_nova', 'bittorrent'] {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.607307 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] osapi_compute_listen = 0.0.0.0 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.608016 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] osapi_compute_listen_port = 8774 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.610001 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] osapi_compute_unique_server_name_scope = {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.610119 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writer thread running {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:99}} Aug 18 21:38:17.610786 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] osapi_compute_workers = 2 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.616313 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] password_length = 12 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.617130 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] periodic_enable = True {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.617777 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] periodic_fuzzy_delay = 60 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.618431 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] pointer_model = usbtablet {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.619098 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] preallocate_images = none {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.619738 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] publish_errors = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.620299 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] pybasedir = /opt/stack/nova {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.622247 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] ram_allocation_ratio = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.622874 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] rate_limit_burst = 0 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.625428 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] rate_limit_except_level = CRITICAL {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.625428 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] rate_limit_interval = 0 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.625428 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] reboot_timeout = 0 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.625428 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] reclaim_instance_interval = 0 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.625428 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] record = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.626379 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] report_interval = 10 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.627020 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] rescue_timeout = 0 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.627647 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] reserved_host_cpus = 0 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.628289 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] reserved_host_disk_mb = 0 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.628919 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] reserved_host_memory_mb = 512 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.629576 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] reserved_huge_pages = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.630194 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] resize_confirm_window = 0 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.630841 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] resize_fs_using_block_device = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.631463 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] resume_guests_state_on_host_boot = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.632126 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] rootwrap_config = /etc/nova/rootwrap.conf {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.632764 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] rpc_response_timeout = 60 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.633371 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] run_external_periodic_tasks = True {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.634049 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] running_deleted_instance_action = reap {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.634743 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] running_deleted_instance_poll_interval = 1800 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.635366 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] running_deleted_instance_timeout = 0 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.636016 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] scheduler_instance_sync_interval = 120 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.636660 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] service_down_time = 60 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.637436 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] servicegroup_driver = db {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.638056 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] shelved_offload_time = 0 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.638672 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] shelved_poll_interval = 3600 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.639334 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] shutdown_timeout = 0 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.639915 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] source_is_ipv6 = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.640382 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] ssl_only = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.640982 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] state_path = /opt/stack/data/nova {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.641565 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] sync_power_state_interval = 600 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.642164 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] sync_power_state_pool_size = 1000 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.642763 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] syslog_log_facility = LOG_USER {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.643356 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] tempdir = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.643976 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] timeout_nbd = 10 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.644583 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] transport_url = **** {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.645146 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] update_resources_interval = 0 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.645787 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] use_cow_images = True {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.646339 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] use_eventlog = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.646898 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] use_journal = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.647482 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] use_json = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.648075 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] use_rootwrap_daemon = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.648641 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] use_stderr = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.649200 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] use_syslog = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.649745 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] vcpu_pin_set = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.650310 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] vif_plugging_is_fatal = True {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.650905 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] vif_plugging_timeout = 300 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.651505 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] virt_mkfs = [] {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.652084 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] volume_usage_poll_interval = 0 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.652694 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] watch_log_file = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.653268 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] web = /usr/share/spice-html5 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2602}} Aug 18 21:38:17.653943 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] oslo_concurrency.disable_process_locking = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.654711 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] oslo_concurrency.lock_path = /opt/stack/data/nova {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.655330 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] oslo_messaging_metrics.metrics_buffer_size = 1000 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.655924 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] oslo_messaging_metrics.metrics_enabled = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.656499 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] oslo_messaging_metrics.metrics_process_name = {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.657062 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] oslo_messaging_metrics.metrics_socket_file = /var/tmp/metrics_collector.sock {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.658134 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] oslo_messaging_metrics.metrics_thread_stop_timeout = 10 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.658807 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] api.auth_strategy = keystone {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.659390 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] api.compute_link_prefix = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.660002 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] api.config_drive_skip_versions = 1.0 2007-01-19 2007-03-01 2007-08-29 2007-10-10 2007-12-15 2008-02-01 2008-09-01 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.660570 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] api.dhcp_domain = novalocal {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.661150 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] api.enable_instance_password = True {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.661736 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] api.glance_link_prefix = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.662306 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] api.instance_list_cells_batch_fixed_size = 100 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.662871 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] api.instance_list_cells_batch_strategy = distributed {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.663425 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] api.instance_list_per_project_cells = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.664079 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] api.list_records_by_skipping_down_cells = True {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.664819 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] api.local_metadata_per_cell = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.665406 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] api.max_limit = 1000 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.665974 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] api.metadata_cache_expiration = 15 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.666553 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] api.neutron_default_tenant_id = default {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.667129 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] api.use_forwarded_for = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.667898 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] api.use_neutron_default_nets = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.669241 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] api.vendordata_dynamic_connect_timeout = 5 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.670107 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] api.vendordata_dynamic_failure_fatal = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.670670 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] api.vendordata_dynamic_read_timeout = 5 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.671272 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] api.vendordata_dynamic_ssl_certfile = {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.672103 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] api.vendordata_dynamic_targets = [] {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.672867 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] api.vendordata_jsonfile_path = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.673638 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] api.vendordata_providers = ['StaticJSON'] {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.674430 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] cache.backend = dogpile.cache.memcached {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.675216 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] cache.backend_argument = **** {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.675912 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] cache.config_prefix = cache.oslo {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.676729 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] cache.dead_timeout = 60.0 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.677345 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] cache.debug_cache_backend = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.678409 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] cache.enable_retry_client = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.679016 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] cache.enable_socket_keepalive = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.679699 n-d-853704-1 nova-conductor[120657]: DEBUG nova.service [None req-e877d9a8-54e5-4cd5-b86d-e17960e5fb33 None None] Creating RPC server for service conductor {{(pid=120657) start /opt/stack/nova/nova/service.py:182}} Aug 18 21:38:17.679999 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] cache.enabled = True {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.680670 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] cache.expiration_time = 600 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.681460 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] cache.hashclient_retry_attempts = 2 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.682069 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] cache.hashclient_retry_delay = 1.0 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.682830 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] cache.memcache_dead_retry = 300 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.683482 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] cache.memcache_pool_connection_get_timeout = 10 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.684927 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] cache.memcache_pool_flush_on_reconnect = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.685552 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] cache.memcache_pool_maxsize = 10 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.686302 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] cache.memcache_pool_unused_timeout = 60 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.686941 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] cache.memcache_servers = ['localhost:11211'] {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.687760 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] cache.memcache_socket_timeout = 1.0 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.688535 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] cache.proxies = [] {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.689334 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] cache.retry_attempts = 2 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.690002 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] cache.retry_delay = 0.0 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.690749 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] cache.socket_keepalive_count = 1 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.691352 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] cache.socket_keepalive_idle = 1 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.692116 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] cache.socket_keepalive_interval = 1 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.692722 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] cache.tls_allowed_ciphers = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.693556 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] cache.tls_cafile = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.694206 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] cache.tls_certfile = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.694963 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] cache.tls_enabled = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.695551 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] cache.tls_keyfile = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.696399 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] cinder.auth_section = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.697000 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] cinder.auth_type = password {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.697579 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] cinder.cafile = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.698223 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] cinder.catalog_info = volumev3::publicURL {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.698874 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] cinder.certfile = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.699530 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] cinder.collect_timing = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.700213 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] cinder.cross_az_attach = True {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.700792 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] cinder.debug = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.701341 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] cinder.endpoint_template = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.701956 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] cinder.http_retries = 3 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.702510 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] cinder.insecure = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.703073 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] cinder.keyfile = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.703645 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] cinder.os_region_name = RegionOne {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.704218 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] cinder.split_loggers = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.704846 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] cinder.timeout = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.705416 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] compute.consecutive_build_service_disable_threshold = 10 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.705971 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] compute.cpu_dedicated_set = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.706564 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] compute.cpu_shared_set = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.707268 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] compute.image_type_exclude_list = [] {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.708882 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] compute.live_migration_wait_for_vif_plug = True {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.709448 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] compute.max_concurrent_disk_ops = 0 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.710068 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] compute.max_disk_devices_to_attach = -1 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.710916 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] compute.packing_host_numa_cells_allocation_strategy = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.711574 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] compute.provider_config_location = /etc/nova/provider_config/ {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.713247 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] compute.resource_provider_association_refresh = 300 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.715213 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] compute.shutdown_retry_interval = 10 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.717064 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] conductor.workers = 2 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.717682 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] console.allowed_origins = [] {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.718250 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] console.ssl_ciphers = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.718831 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] console.ssl_minimum_version = default {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.719417 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] consoleauth.token_ttl = 600 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.720068 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] cyborg.cafile = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.720666 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] cyborg.certfile = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.721264 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] cyborg.collect_timing = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.721850 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] cyborg.connect_retries = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.722429 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] cyborg.connect_retry_delay = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.723005 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] cyborg.endpoint_override = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.723623 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] cyborg.insecure = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.724254 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] cyborg.keyfile = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.724827 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] cyborg.max_version = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.725411 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] cyborg.min_version = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.725754 n-d-853704-1 nova-conductor[120658]: DEBUG nova.service [None req-941b5ded-1cec-4e4b-8ef3-7538d164155f None None] Creating RPC server for service conductor {{(pid=120658) start /opt/stack/nova/nova/service.py:182}} Aug 18 21:38:17.726186 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] cyborg.region_name = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.726853 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] cyborg.service_name = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.727446 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] cyborg.service_type = accelerator {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.728047 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] cyborg.split_loggers = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.728627 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] cyborg.status_code_retries = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.729388 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] cyborg.status_code_retry_delay = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.729998 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] cyborg.timeout = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.731549 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] cyborg.valid_interfaces = ['internal', 'public'] {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.732401 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] cyborg.version = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.733224 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] database.backend = sqlalchemy {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.736861 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] database.connection = **** {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.739075 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] database.connection_debug = 0 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.740071 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] database.connection_parameters = {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.746043 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] database.connection_recycle_time = 3600 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.751436 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] database.connection_trace = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.752672 n-d-853704-1 nova-conductor[120657]: DEBUG nova.service [None req-e877d9a8-54e5-4cd5-b86d-e17960e5fb33 None None] Join ServiceGroup membership for this service conductor {{(pid=120657) start /opt/stack/nova/nova/service.py:199}} Aug 18 21:38:17.753011 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] database.db_inc_retry_interval = True {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.753255 n-d-853704-1 nova-conductor[120657]: DEBUG nova.servicegroup.drivers.db [None req-e877d9a8-54e5-4cd5-b86d-e17960e5fb33 None None] DB_Driver: join new ServiceGroup member n-d-853704-1 to the conductor group, service = {{(pid=120657) join /opt/stack/nova/nova/servicegroup/drivers/db.py:44}} Aug 18 21:38:17.754575 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] database.db_max_retries = 20 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.756016 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] database.db_max_retry_interval = 10 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.756868 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] database.db_retry_interval = 1 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.757678 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] database.max_overflow = 50 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.758518 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] database.max_pool_size = 5 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.759322 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] database.max_retries = 10 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.760146 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] database.mysql_enable_ndb = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.760957 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] database.mysql_sql_mode = TRADITIONAL {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.761826 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] database.pool_timeout = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.763378 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] database.retry_interval = 10 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.764213 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] database.slave_connection = **** {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.765012 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] database.sqlite_synchronous = True {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.765817 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] database.use_db_reconnect = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.766651 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] api_database.backend = sqlalchemy {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.770410 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] api_database.connection = **** {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.770854 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] api_database.connection_debug = 0 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.771516 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] api_database.connection_parameters = {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.772173 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] api_database.connection_recycle_time = 3600 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.772824 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] api_database.connection_trace = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.773453 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] api_database.db_inc_retry_interval = True {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.775851 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] api_database.db_max_retries = 20 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.779009 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] api_database.db_max_retry_interval = 10 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.779707 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] api_database.db_retry_interval = 1 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.780866 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] api_database.max_overflow = 50 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.781478 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] api_database.max_pool_size = 5 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.782103 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] api_database.max_retries = 10 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.782707 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] api_database.mysql_enable_ndb = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.783332 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] api_database.mysql_sql_mode = TRADITIONAL {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.783968 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] api_database.pool_timeout = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.784581 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] api_database.retry_interval = 10 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.785181 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] api_database.slave_connection = **** {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.785789 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] api_database.sqlite_synchronous = True {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.786440 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] devices.enabled_mdev_types = [] {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.788206 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] ephemeral_storage_encryption.cipher = aes-xts-plain64 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.789565 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] ephemeral_storage_encryption.enabled = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.791260 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] ephemeral_storage_encryption.key_size = 512 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.791995 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] glance.api_servers = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.792843 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] glance.cafile = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.794045 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] glance.certfile = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.794670 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] glance.collect_timing = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.796842 n-d-853704-1 nova-conductor[120658]: DEBUG nova.service [None req-941b5ded-1cec-4e4b-8ef3-7538d164155f None None] Join ServiceGroup membership for this service conductor {{(pid=120658) start /opt/stack/nova/nova/service.py:199}} Aug 18 21:38:17.797304 n-d-853704-1 nova-conductor[120658]: DEBUG nova.servicegroup.drivers.db [None req-941b5ded-1cec-4e4b-8ef3-7538d164155f None None] DB_Driver: join new ServiceGroup member n-d-853704-1 to the conductor group, service = {{(pid=120658) join /opt/stack/nova/nova/servicegroup/drivers/db.py:44}} Aug 18 21:38:17.798583 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] glance.connect_retries = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.799205 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] glance.connect_retry_delay = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.799873 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] glance.debug = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.800567 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] glance.default_trusted_certificate_ids = [] {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.801177 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] glance.enable_certificate_validation = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.801787 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] glance.enable_rbd_download = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.802388 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] glance.endpoint_override = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.803245 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] glance.insecure = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.803879 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] glance.keyfile = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.804478 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] glance.max_version = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.805089 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] glance.min_version = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.805716 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] glance.num_retries = 3 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.806328 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] glance.rbd_ceph_conf = {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.806935 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] glance.rbd_connect_timeout = 5 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.807542 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] glance.rbd_pool = {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.808290 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] glance.rbd_user = {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.809526 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] glance.region_name = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.810380 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] glance.service_name = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.811025 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] glance.service_type = image {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.811638 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] glance.split_loggers = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.812260 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] glance.status_code_retries = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.812861 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] glance.status_code_retry_delay = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.813455 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] glance.timeout = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.814150 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] glance.valid_interfaces = ['internal', 'public'] {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.814739 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] glance.verify_glance_signatures = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.815381 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] glance.version = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.816074 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] guestfs.debug = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.817125 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] hyperv.config_drive_cdrom = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.817716 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] hyperv.config_drive_inject_password = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.818229 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] hyperv.dynamic_memory_ratio = 1.0 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.818651 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] hyperv.enable_instance_metrics_collection = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.819057 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] hyperv.enable_remotefx = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.819464 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] hyperv.instances_path_share = {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.819901 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] hyperv.iscsi_initiator_list = [] {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.820298 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] hyperv.limit_cpu_features = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.820691 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] hyperv.mounted_disk_query_retry_count = 10 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.821083 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] hyperv.mounted_disk_query_retry_interval = 5 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.821483 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] hyperv.power_state_check_timeframe = 60 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.821879 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] hyperv.power_state_event_polling_interval = 2 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.822310 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] hyperv.qemu_img_cmd = qemu-img.exe {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.822813 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] hyperv.use_multipath_io = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.823219 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] hyperv.volume_attach_retry_count = 10 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.823614 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] hyperv.volume_attach_retry_interval = 5 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.824025 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] hyperv.vswitch_name = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.825174 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] hyperv.wait_soft_reboot_seconds = 60 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.825788 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] mks.enabled = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.826944 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] mks.mksproxy_base_url = http://127.0.0.1:6090/ {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.827390 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] image_cache.manager_interval = 2400 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.827896 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] image_cache.precache_concurrency = 1 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.828342 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] image_cache.remove_unused_base_images = True {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.828752 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] image_cache.remove_unused_original_minimum_age_seconds = 86400 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.829160 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] image_cache.remove_unused_resized_minimum_age_seconds = 3600 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.829580 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] image_cache.subdirectory_name = _base {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.830027 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] ironic.api_max_retries = 60 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.830427 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] ironic.api_retry_interval = 2 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.830832 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] ironic.auth_section = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.831227 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] ironic.auth_type = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.831611 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] ironic.cafile = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.832022 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] ironic.certfile = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.832419 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] ironic.collect_timing = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.832810 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] ironic.connect_retries = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.833207 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] ironic.connect_retry_delay = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.833592 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] ironic.endpoint_override = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.833980 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] ironic.insecure = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.834364 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] ironic.keyfile = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.834746 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] ironic.max_version = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.835137 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] ironic.min_version = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.835528 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] ironic.partition_key = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.835952 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] ironic.peer_list = [] {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.836341 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] ironic.region_name = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.836730 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] ironic.serial_console_state_timeout = 10 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.837110 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] ironic.service_name = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.837615 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] ironic.service_type = baremetal {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.838015 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] ironic.split_loggers = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.838399 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] ironic.status_code_retries = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.838788 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] ironic.status_code_retry_delay = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.839192 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] ironic.timeout = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.839634 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] ironic.valid_interfaces = ['internal', 'public'] {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.840125 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] ironic.version = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.840621 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] key_manager.backend = nova.keymgr.conf_key_mgr.ConfKeyManager {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.841052 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] key_manager.fixed_key = **** {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.841538 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] barbican.auth_endpoint = http://localhost/identity/v3 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.841924 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] barbican.barbican_api_version = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.842346 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] barbican.barbican_endpoint = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.842788 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] barbican.barbican_endpoint_type = public {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.843205 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] barbican.barbican_region_name = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.843617 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] barbican.cafile = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.844041 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] barbican.certfile = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.844459 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] barbican.collect_timing = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.844855 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] barbican.insecure = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.845262 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] barbican.keyfile = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.845673 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] barbican.number_of_retries = 60 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.846062 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] barbican.retry_delay = 1 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.846461 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] barbican.send_service_user_token = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.846851 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] barbican.split_loggers = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.847243 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] barbican.timeout = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.847637 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] barbican.verify_ssl = True {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.848045 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] barbican.verify_ssl_path = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.848461 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] barbican_service_user.auth_section = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.848869 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] barbican_service_user.auth_type = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.849251 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] barbican_service_user.cafile = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.849637 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] barbican_service_user.certfile = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.850027 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] barbican_service_user.collect_timing = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.850411 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] barbican_service_user.insecure = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.850816 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] barbican_service_user.keyfile = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.851205 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] barbican_service_user.split_loggers = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.851599 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] barbican_service_user.timeout = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.852058 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] vault.approle_role_id = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.852450 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] vault.approle_secret_id = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.852836 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] vault.cafile = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.853218 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] vault.certfile = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.853609 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] vault.collect_timing = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.853996 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] vault.insecure = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.854377 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] vault.keyfile = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.854788 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] vault.kv_mountpoint = secret {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.855191 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] vault.kv_version = 2 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.855646 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] vault.namespace = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.856202 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] vault.root_token_id = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.856607 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] vault.split_loggers = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.857005 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] vault.ssl_ca_crt_file = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.857386 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] vault.timeout = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.857774 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] vault.use_ssl = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.858212 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] vault.vault_url = http://127.0.0.1:8200 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.858656 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] keystone.cafile = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.859038 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] keystone.certfile = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.859423 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] keystone.collect_timing = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.859902 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] keystone.connect_retries = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.860288 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] keystone.connect_retry_delay = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.860670 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] keystone.endpoint_override = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.861057 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] keystone.insecure = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.861439 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] keystone.keyfile = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.861821 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] keystone.max_version = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.862228 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] keystone.min_version = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.862620 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] keystone.region_name = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.863000 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] keystone.service_name = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.863396 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] keystone.service_type = identity {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.863815 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] keystone.split_loggers = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.864199 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] keystone.status_code_retries = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.864579 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] keystone.status_code_retry_delay = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.864957 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] keystone.timeout = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.865438 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] keystone.valid_interfaces = ['internal', 'public'] {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.865819 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] keystone.version = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.866343 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] libvirt.connection_uri = {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.866753 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] libvirt.cpu_mode = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.867145 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] libvirt.cpu_model_extra_flags = [] {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.867547 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] libvirt.cpu_models = [] {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.868021 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] libvirt.device_detach_attempts = 8 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.868427 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] libvirt.device_detach_timeout = 20 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.868986 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] libvirt.disk_cachemodes = [] {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.869535 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] libvirt.disk_prefix = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.869936 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] libvirt.enabled_perf_events = [] {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.870330 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] libvirt.file_backed_memory = 0 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.870727 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] libvirt.gid_maps = [] {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.871133 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] libvirt.hw_disk_discard = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.872193 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] libvirt.hw_machine_type = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.872639 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] libvirt.images_rbd_ceph_conf = {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.873086 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] libvirt.images_rbd_glance_copy_poll_interval = 15 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.873500 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] libvirt.images_rbd_glance_copy_timeout = 600 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.873905 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] libvirt.images_rbd_glance_store_name = {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.874329 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] libvirt.images_rbd_pool = rbd {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.874936 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] libvirt.images_type = default {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.875330 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] libvirt.images_volume_group = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.875752 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] libvirt.inject_key = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.876178 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] libvirt.inject_partition = -2 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.876572 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] libvirt.inject_password = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.876987 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] libvirt.iscsi_iface = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.877379 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] libvirt.iser_use_multipath = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.877769 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] libvirt.live_migration_bandwidth = 0 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.878164 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] libvirt.live_migration_completion_timeout = 800 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.878553 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] libvirt.live_migration_downtime = 500 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.878945 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] libvirt.live_migration_downtime_delay = 75 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.879358 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] libvirt.live_migration_downtime_steps = 10 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.879769 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] libvirt.live_migration_inbound_addr = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.880166 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] libvirt.live_migration_permit_auto_converge = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.880584 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] libvirt.live_migration_permit_post_copy = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.880975 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] libvirt.live_migration_scheme = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.881387 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] libvirt.live_migration_timeout_action = abort {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.881776 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] libvirt.live_migration_tunnelled = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.882179 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] libvirt.live_migration_uri = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.882553 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] libvirt.live_migration_with_native_tls = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.882955 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] libvirt.max_queues = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.883347 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] libvirt.mem_stats_period_seconds = 10 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.883774 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] libvirt.nfs_mount_options = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.884317 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] libvirt.nfs_mount_point_base = /opt/stack/data/nova/mnt {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.884708 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] libvirt.num_aoe_discover_tries = 3 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.885095 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] libvirt.num_iser_scan_tries = 5 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.885501 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] libvirt.num_memory_encrypted_guests = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.885896 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] libvirt.num_nvme_discover_tries = 5 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.886281 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] libvirt.num_pcie_ports = 0 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.886675 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] libvirt.num_volume_scan_tries = 5 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.887847 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] libvirt.pmem_namespaces = [] {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.888294 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] libvirt.quobyte_client_cfg = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.888767 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] libvirt.quobyte_mount_point_base = /opt/stack/data/nova/mnt {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.889163 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] libvirt.rbd_connect_timeout = 5 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.889558 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] libvirt.rbd_destroy_volume_retries = 12 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.889951 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] libvirt.rbd_destroy_volume_retry_interval = 5 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.890337 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] libvirt.rbd_secret_uuid = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.890738 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] libvirt.rbd_user = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.891129 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] libvirt.realtime_scheduler_priority = 1 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.891560 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] libvirt.remote_filesystem_transport = ssh {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.891993 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] libvirt.rescue_image_id = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.892387 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] libvirt.rescue_kernel_id = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.892768 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] libvirt.rescue_ramdisk_id = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.893178 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] libvirt.rng_dev_path = /dev/urandom {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.893568 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] libvirt.rx_queue_size = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.893976 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] libvirt.smbfs_mount_options = {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.894873 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] libvirt.smbfs_mount_point_base = /opt/stack/data/nova/mnt {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.894873 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] libvirt.snapshot_compression = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.895207 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] libvirt.snapshot_image_format = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.895742 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] libvirt.snapshots_directory = /opt/stack/data/nova/instances/snapshots {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.896159 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] libvirt.sparse_logical_volumes = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.896573 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] libvirt.swtpm_enabled = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.897002 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] libvirt.swtpm_group = tss {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.897410 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] libvirt.swtpm_user = tss {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.897818 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] libvirt.sysinfo_serial = unique {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.898240 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] libvirt.tx_queue_size = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.898600 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] libvirt.uid_maps = [] {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.898996 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] libvirt.use_virtio_for_bridges = True {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.899401 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] libvirt.virt_type = kvm {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.899837 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] libvirt.volume_clear = zero {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.900227 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] libvirt.volume_clear_size = 0 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.900694 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] libvirt.volume_use_multipath = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.901009 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] libvirt.vzstorage_cache_path = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.901499 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] libvirt.vzstorage_log_path = /var/log/vstorage/%(cluster_name)s/nova.log.gz {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.901816 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] libvirt.vzstorage_mount_group = qemu {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.902220 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] libvirt.vzstorage_mount_opts = [] {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.902663 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] libvirt.vzstorage_mount_perms = 0770 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.903177 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] libvirt.vzstorage_mount_point_base = /opt/stack/data/nova/mnt {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.903590 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] libvirt.vzstorage_mount_user = stack {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.904023 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] libvirt.wait_soft_reboot_seconds = 120 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.904483 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] neutron.auth_section = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.904895 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] neutron.auth_type = password {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.905282 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] neutron.cafile = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.905666 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] neutron.certfile = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.906063 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] neutron.collect_timing = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.906466 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] neutron.connect_retries = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.906849 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] neutron.connect_retry_delay = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.907252 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] neutron.default_floating_pool = public {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.907635 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] neutron.endpoint_override = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.908068 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] neutron.extension_sync_interval = 600 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.908457 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] neutron.http_retries = 3 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.908846 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] neutron.insecure = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.909230 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] neutron.keyfile = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.909612 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] neutron.max_version = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.910017 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] neutron.metadata_proxy_shared_secret = **** {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.910406 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] neutron.min_version = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.910806 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] neutron.ovs_bridge = br-int {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.911198 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] neutron.physnets = [] {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.911601 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] neutron.region_name = RegionOne {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.912023 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] neutron.service_metadata_proxy = True {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.912419 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] neutron.service_name = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.912832 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] neutron.service_type = network {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.913216 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] neutron.split_loggers = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.913595 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] neutron.status_code_retries = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.913976 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] neutron.status_code_retry_delay = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.914373 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] neutron.timeout = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.914799 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] neutron.valid_interfaces = ['internal', 'public'] {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.915189 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] neutron.version = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.915596 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] notifications.bdms_in_notifications = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.916073 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] notifications.default_level = INFO {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.916489 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] notifications.notification_format = unversioned {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.916917 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] notifications.notify_on_state_change = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.917340 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] notifications.versioned_notifications_topics = ['versioned_notifications'] {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.917767 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] pci.alias = [] {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.918252 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] pci.device_spec = [] {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.918778 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] placement.auth_section = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.919128 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] placement.auth_type = password {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.919573 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] placement.auth_url = http://10.222.0.49/identity {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.920012 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] placement.cafile = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.920519 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] placement.certfile = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.920925 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] placement.collect_timing = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.921309 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] placement.connect_retries = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.921697 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] placement.connect_retry_delay = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.922109 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] placement.default_domain_id = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.922496 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] placement.default_domain_name = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.922994 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] placement.domain_id = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.923388 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] placement.domain_name = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.923801 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] placement.endpoint_override = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.924190 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] placement.insecure = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.924603 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] placement.keyfile = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.924993 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] placement.max_version = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.925396 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] placement.min_version = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.925798 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] placement.password = **** {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.926183 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] placement.project_domain_id = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.926581 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] placement.project_domain_name = Default {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.926984 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] placement.project_id = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.927398 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] placement.project_name = service {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.928141 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] placement.region_name = RegionOne {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.928543 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] placement.service_name = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.928951 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] placement.service_type = placement {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.929345 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] placement.split_loggers = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.929729 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] placement.status_code_retries = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.930382 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] placement.status_code_retry_delay = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.930775 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] placement.system_scope = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.931166 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] placement.timeout = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.931553 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] placement.trust_id = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.931973 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] placement.user_domain_id = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.932383 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] placement.user_domain_name = Default {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.932777 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] placement.user_id = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.933208 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] placement.username = placement {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.933633 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] placement.valid_interfaces = ['internal', 'public'] {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.934678 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] placement.version = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.935264 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] quota.cores = 20 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.935834 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] quota.count_usage_from_placement = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.936407 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] quota.driver = nova.quota.DbQuotaDriver {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.937001 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] quota.injected_file_content_bytes = 10240 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.937619 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] quota.injected_file_path_length = 255 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.938163 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] quota.injected_files = 5 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.938722 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] quota.instances = 10 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.939315 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] quota.key_pairs = 100 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.939925 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] quota.metadata_items = 128 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.940465 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] quota.ram = 51200 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.940867 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] quota.recheck_quota = True {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.941271 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] quota.server_group_members = 10 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.941668 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] quota.server_groups = 10 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.942072 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] rdp.enabled = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.942853 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] rdp.html5_proxy_base_url = http://127.0.0.1:6083/ {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.943167 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] scheduler.discover_hosts_in_cells_interval = -1 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.943605 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] scheduler.enable_isolated_aggregate_filtering = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.944053 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] scheduler.image_metadata_prefilter = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.944516 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] scheduler.limit_tenants_to_placement_aggregate = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.944935 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] scheduler.max_attempts = 3 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.945346 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] scheduler.max_placement_results = 1000 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.945784 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] scheduler.placement_aggregate_required_for_tenants = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.946227 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] scheduler.query_placement_for_availability_zone = True {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.946683 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] scheduler.query_placement_for_image_type_support = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.947113 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] scheduler.query_placement_for_routed_network_aggregates = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.947540 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] scheduler.workers = 2 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.948038 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] filter_scheduler.aggregate_image_properties_isolation_namespace = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.948522 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] filter_scheduler.aggregate_image_properties_isolation_separator = . {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.948907 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] filter_scheduler.available_filters = ['nova.scheduler.filters.all_filters'] {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.949414 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] filter_scheduler.build_failure_weight_multiplier = 1000000.0 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.950301 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] filter_scheduler.cpu_weight_multiplier = 1.0 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.950708 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] filter_scheduler.cross_cell_move_weight_multiplier = 1000000.0 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.951104 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] filter_scheduler.disk_weight_multiplier = 1.0 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.951581 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] filter_scheduler.enabled_filters = ['AvailabilityZoneFilter', 'ComputeFilter', 'ComputeCapabilitiesFilter', 'ImagePropertiesFilter', 'ServerGroupAntiAffinityFilter', 'ServerGroupAffinityFilter', 'SameHostFilter', 'DifferentHostFilter'] {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.952005 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] filter_scheduler.host_subset_size = 1 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.952393 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] filter_scheduler.image_properties_default_architecture = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.952796 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] filter_scheduler.io_ops_weight_multiplier = -1.0 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.953220 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] filter_scheduler.isolated_hosts = [] {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.953605 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] filter_scheduler.isolated_images = [] {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.954041 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] filter_scheduler.max_instances_per_host = 50 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.954438 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] filter_scheduler.max_io_ops_per_host = 8 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.954836 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] filter_scheduler.pci_weight_multiplier = 1.0 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.955238 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] filter_scheduler.ram_weight_multiplier = 1.0 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.955725 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] filter_scheduler.restrict_isolated_hosts_to_isolated_images = True {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.956104 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] filter_scheduler.shuffle_best_same_weighed_hosts = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.956501 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] filter_scheduler.soft_affinity_weight_multiplier = 1.0 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.956917 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] filter_scheduler.soft_anti_affinity_weight_multiplier = 1.0 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.957347 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] filter_scheduler.track_instance_changes = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.957771 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] filter_scheduler.weight_classes = ['nova.scheduler.weights.all_weighers'] {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.958207 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] metrics.required = True {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.958602 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] metrics.weight_multiplier = 1.0 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.959645 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] metrics.weight_of_unavailable = -10000.0 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.959645 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] metrics.weight_setting = [] {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.960030 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] serial_console.base_url = ws://127.0.0.1:6083/ {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.960432 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] serial_console.enabled = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.960876 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] serial_console.port_range = 10000:20000 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.961284 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] serial_console.proxyclient_address = 127.0.0.1 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.961699 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] serial_console.serialproxy_host = 0.0.0.0 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.962129 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] serial_console.serialproxy_port = 6083 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.962532 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] service_user.auth_section = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.963025 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] service_user.auth_type = password {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.963415 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] service_user.cafile = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.963828 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] service_user.certfile = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.964221 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] service_user.collect_timing = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.964613 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] service_user.insecure = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.965019 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] service_user.keyfile = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.965423 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] service_user.send_service_user_token = True {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.965813 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] service_user.split_loggers = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.966196 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] service_user.timeout = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.966634 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] spice.agent_enabled = True {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.967024 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] spice.enabled = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.967703 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] spice.html5proxy_base_url = http://127.0.0.1:6082/spice_auto.html {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.968293 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] spice.html5proxy_host = 0.0.0.0 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.968745 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] spice.html5proxy_port = 6082 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.969158 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] spice.server_listen = 127.0.0.1 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.969569 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] spice.server_proxyclient_address = 127.0.0.1 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.969967 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] upgrade_levels.baseapi = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.970359 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] upgrade_levels.cert = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.970766 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] upgrade_levels.compute = auto {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.971174 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] upgrade_levels.conductor = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.971559 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] upgrade_levels.scheduler = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.972009 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] vendordata_dynamic_auth.auth_section = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.972426 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] vendordata_dynamic_auth.auth_type = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.972824 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] vendordata_dynamic_auth.cafile = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.973211 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] vendordata_dynamic_auth.certfile = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.973661 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] vendordata_dynamic_auth.collect_timing = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.974053 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] vendordata_dynamic_auth.insecure = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.974438 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] vendordata_dynamic_auth.keyfile = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.974963 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] vendordata_dynamic_auth.split_loggers = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.975357 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] vendordata_dynamic_auth.timeout = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.975837 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] vmware.api_retry_count = 10 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.976244 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] vmware.ca_file = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.976655 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] vmware.cache_prefix = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.977042 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] vmware.cluster_name = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.977430 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] vmware.connection_pool_size = 10 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.977813 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] vmware.console_delay_seconds = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.978195 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] vmware.datastore_regex = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.978576 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] vmware.host_ip = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.978958 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] vmware.host_password = **** {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.979350 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] vmware.host_port = 443 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.979759 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] vmware.host_username = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.980140 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] vmware.insecure = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.980582 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] vmware.integration_bridge = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.980968 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] vmware.maximum_objects = 100 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.981351 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] vmware.pbm_default_policy = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.981740 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] vmware.pbm_enabled = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.982198 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] vmware.pbm_wsdl_location = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.982523 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] vmware.serial_log_dir = /opt/vmware/vspc {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.982905 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] vmware.serial_port_proxy_uri = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.983285 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] vmware.serial_port_service_uri = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.983688 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] vmware.task_poll_interval = 0.5 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.984089 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] vmware.use_linked_clone = True {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.984507 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] vmware.vnc_keymap = en-us {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.984879 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] vmware.vnc_port = 5900 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.985310 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] vmware.vnc_port_total = 10000 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.985763 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] vnc.auth_schemes = ['none'] {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.986162 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] vnc.enabled = True {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.986759 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] vnc.novncproxy_base_url = http://127.0.0.1:6080/vnc_auto.html {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.987193 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] vnc.novncproxy_host = 0.0.0.0 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.987598 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] vnc.novncproxy_port = 6080 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.988037 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] vnc.server_listen = 127.0.0.1 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.988446 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] vnc.server_proxyclient_address = 127.0.0.1 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.988852 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] vnc.vencrypt_ca_certs = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.989239 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] vnc.vencrypt_client_cert = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.989639 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] vnc.vencrypt_client_key = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.990058 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] workarounds.disable_compute_service_check_for_ffu = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.990457 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] workarounds.disable_fallback_pcpu_query = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.990849 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] workarounds.disable_group_policy_check_upcall = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.991282 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] workarounds.disable_libvirt_livesnapshot = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.991635 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] workarounds.disable_rootwrap = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.992052 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] workarounds.enable_numa_live_migration = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.992469 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] workarounds.enable_qemu_monitor_announce_self = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.992862 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] workarounds.ensure_libvirt_rbd_instance_dir_cleanup = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.993259 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] workarounds.handle_virt_lifecycle_events = True {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.993665 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] workarounds.libvirt_disable_apic = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.994038 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] workarounds.never_download_image_if_on_rbd = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.994430 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] workarounds.reserve_disk_resource_for_image_cache = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.994828 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] workarounds.skip_cpu_compare_on_dest = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.995241 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] workarounds.skip_hypervisor_version_check_on_lm = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.995629 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] workarounds.unified_limits_count_pcpu_as_vcpu = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.996050 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] workarounds.wait_for_vif_plugged_event_during_hard_reboot = [] {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.997103 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] wsgi.api_paste_config = /etc/nova/api-paste.ini {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.997527 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] wsgi.client_socket_timeout = 900 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.997928 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] wsgi.default_pool_size = 1000 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.998334 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] wsgi.keep_alive = True {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.998784 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] wsgi.max_header_line = 16384 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.999177 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] wsgi.secure_proxy_ssl_header = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:17.999568 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] wsgi.ssl_ca_file = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:18.000046 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] wsgi.ssl_cert_file = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:18.000395 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] wsgi.ssl_key_file = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:18.000796 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] wsgi.tcp_keepidle = 600 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:18.001216 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] wsgi.wsgi_log_format = %(client_ip)s "%(request_line)s" status: %(status_code)s len: %(body_length)s time: %(wall_seconds).7f {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:18.001616 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] zvm.ca_file = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:18.002007 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] zvm.cloud_connector_url = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:18.002531 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] zvm.image_tmp_path = /opt/stack/data/nova/images {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:18.002938 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] zvm.reachable_timeout = 300 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:18.003398 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] oslo_policy.enforce_new_defaults = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:18.003831 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] oslo_policy.enforce_scope = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:18.004261 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] oslo_policy.policy_default_rule = default {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:18.004714 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] oslo_policy.policy_dirs = ['policy.d'] {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:18.005134 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] oslo_policy.policy_file = policy.yaml {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:18.005602 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] oslo_policy.remote_content_type = application/x-www-form-urlencoded {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:18.006014 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] oslo_policy.remote_ssl_ca_crt_file = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:18.006421 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] oslo_policy.remote_ssl_client_crt_file = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:18.006835 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] oslo_policy.remote_ssl_client_key_file = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:18.007229 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] oslo_policy.remote_ssl_verify_server_crt = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:18.007671 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] oslo_versionedobjects.fatal_exception_format_errors = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:18.008110 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] oslo_middleware.http_basic_auth_user_file = /etc/htpasswd {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:18.008560 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] profiler.connection_string = messaging:// {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:18.009524 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] profiler.enabled = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:18.009524 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] profiler.es_doc_type = notification {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:18.009852 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] profiler.es_scroll_size = 10000 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:18.010255 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] profiler.es_scroll_time = 2m {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:18.010644 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] profiler.filter_error_trace = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:18.011069 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] profiler.hmac_keys = SECRET_KEY {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:18.011472 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] profiler.sentinel_service_name = mymaster {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:18.012624 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] profiler.socket_timeout = 0.1 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:18.012849 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] profiler.trace_sqlalchemy = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:18.013292 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] remote_debug.host = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:18.013704 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] remote_debug.port = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:18.014142 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] oslo_messaging_rabbit.amqp_auto_delete = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:18.014568 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] oslo_messaging_rabbit.amqp_durable_queues = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:18.014967 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] oslo_messaging_rabbit.conn_pool_min_size = 2 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:18.015411 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] oslo_messaging_rabbit.conn_pool_ttl = 1200 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:18.015790 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] oslo_messaging_rabbit.direct_mandatory_flag = True {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:18.016188 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] oslo_messaging_rabbit.enable_cancel_on_failover = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:18.016594 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] oslo_messaging_rabbit.heartbeat_in_pthread = True {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:18.017061 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] oslo_messaging_rabbit.heartbeat_rate = 3 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:18.017469 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] oslo_messaging_rabbit.heartbeat_timeout_threshold = 600 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:18.017855 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] oslo_messaging_rabbit.kombu_compression = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:18.018270 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] oslo_messaging_rabbit.kombu_failover_strategy = round-robin {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:18.018707 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] oslo_messaging_rabbit.kombu_missing_consumer_retry_timeout = 60 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:18.019114 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] oslo_messaging_rabbit.kombu_reconnect_delay = 1.0 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:18.019515 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] oslo_messaging_rabbit.rabbit_ha_queues = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:18.019933 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] oslo_messaging_rabbit.rabbit_interval_max = 30 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:18.020361 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] oslo_messaging_rabbit.rabbit_login_method = AMQPLAIN {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:18.020753 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] oslo_messaging_rabbit.rabbit_qos_prefetch_count = 0 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:18.021146 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] oslo_messaging_rabbit.rabbit_quorum_delivery_limit = 0 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:18.021543 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] oslo_messaging_rabbit.rabbit_quorum_queue = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:18.021945 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] oslo_messaging_rabbit.rabbit_quroum_max_memory_bytes = 0 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:18.022336 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] oslo_messaging_rabbit.rabbit_quroum_max_memory_length = 0 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:18.022882 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] oslo_messaging_rabbit.rabbit_retry_backoff = 2 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:18.023283 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] oslo_messaging_rabbit.rabbit_retry_interval = 1 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:18.024303 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] oslo_messaging_rabbit.rabbit_transient_queues_ttl = 1800 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:18.024303 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] oslo_messaging_rabbit.rpc_conn_pool_size = 30 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:18.024693 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] oslo_messaging_rabbit.ssl = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:18.024967 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] oslo_messaging_rabbit.ssl_ca_file = {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:18.025377 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] oslo_messaging_rabbit.ssl_cert_file = {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:18.025834 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] oslo_messaging_rabbit.ssl_enforce_fips_mode = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:18.026258 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] oslo_messaging_rabbit.ssl_key_file = {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:18.026670 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] oslo_messaging_rabbit.ssl_version = {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:18.027125 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] oslo_messaging_notifications.driver = ['messagingv2'] {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:18.027562 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] oslo_messaging_notifications.retry = -1 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:18.028148 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] oslo_messaging_notifications.topics = ['notifications'] {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:18.028528 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] oslo_messaging_notifications.transport_url = **** {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:18.028981 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] oslo_limit.auth_section = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:18.029376 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] oslo_limit.auth_type = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:18.029764 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] oslo_limit.cafile = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:18.030151 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] oslo_limit.certfile = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:18.030570 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] oslo_limit.collect_timing = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:18.030930 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] oslo_limit.connect_retries = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:18.031494 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] oslo_limit.connect_retry_delay = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:18.032324 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] oslo_limit.endpoint_id = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:18.032720 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] oslo_limit.endpoint_override = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:18.033156 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] oslo_limit.insecure = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:18.033507 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] oslo_limit.keyfile = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:18.033922 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] oslo_limit.max_version = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:18.034309 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] oslo_limit.min_version = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:18.034694 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] oslo_limit.region_name = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:18.035091 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] oslo_limit.service_name = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:18.035503 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] oslo_limit.service_type = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:18.035922 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] oslo_limit.split_loggers = False {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:18.036308 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] oslo_limit.status_code_retries = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:18.036695 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] oslo_limit.status_code_retry_delay = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:18.037118 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] oslo_limit.timeout = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:18.037509 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] oslo_limit.valid_interfaces = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:18.037901 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] oslo_limit.version = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:18.038351 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] oslo_reports.file_event_handler = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:18.038749 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] oslo_reports.file_event_handler_interval = 1 {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:18.039159 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] oslo_reports.log_dir = None {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2609}} Aug 18 21:38:18.039487 n-d-853704-1 nova-conductor[119706]: DEBUG oslo_service.service [None req-3a291c53-58ce-4ff7-a421-49582441996a None None] ******************************************************************************** {{(pid=119706) log_opt_values /usr/local/lib/python3.8/dist-packages/oslo_config/cfg.py:2613}} Aug 18 21:38:27.352501 n-d-853704-1 nova-conductor[119706]: DEBUG dbcounter [-] [119706] Writing DB stats nova_api:SELECT=3 {{(pid=119706) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:38:27.516306 n-d-853704-1 nova-conductor[119706]: DEBUG dbcounter [-] [119706] Writing DB stats nova_cell1:SELECT=3 {{(pid=119706) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:38:27.524953 n-d-853704-1 nova-conductor[119706]: DEBUG dbcounter [-] [119706] Writing DB stats nova_cell0:SELECT=3 {{(pid=119706) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:38:42.811861 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=14,nova_cell0:INSERT=1,nova_cell0:UPDATE=2 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:38:52.751532 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=15,nova_cell0:INSERT=1,nova_cell0:UPDATE=3 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:39:12.758550 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=3,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:39:12.810166 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=6,nova_cell0:UPDATE=3 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:39:32.763146 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=4,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:39:52.804073 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:40:12.766969 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:40:32.796589 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:40:52.768716 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:41:02.803774 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=5,nova_cell0:UPDATE=3 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:41:22.778769 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=6,nova_cell0:UPDATE=3 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:41:42.812517 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:42:02.786087 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:42:22.814652 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:42:32.788281 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=5,nova_cell0:UPDATE=3 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:42:42.823565 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=3,nova_cell0:UPDATE=2 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:43:12.790204 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:43:22.821721 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:43:42.797869 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=5,nova_cell0:UPDATE=3 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:43:42.828503 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=3,nova_cell0:UPDATE=2 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:44:02.804147 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=4,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:44:02.834249 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=4,nova_cell0:UPDATE=2 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:44:32.808153 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=6,nova_cell0:UPDATE=3 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:44:32.838385 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=6,nova_cell0:UPDATE=3 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:44:52.815199 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=4,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:44:52.846604 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=4,nova_cell0:UPDATE=2 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:45:12.822728 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=4,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:45:12.854443 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=4,nova_cell0:UPDATE=2 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:45:32.829423 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=4,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:45:32.860092 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=4,nova_cell0:UPDATE=2 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:46:12.817872 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:46:12.858172 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:46:52.824634 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=7,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:46:52.865690 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:47:22.829790 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=6,nova_cell0:UPDATE=3 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:47:32.872521 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:47:52.784245 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_db.sqlalchemy.engines [None req-aaaa146e-bbe3-4fed-b0f5-2adbba765925 tempest-AggregatesAdminTestJSON-959991073 tempest-AggregatesAdminTestJSON-959991073-project] Parent process 119706 forked (120657) with an open database connection, which is being discarded and recreated. {{(pid=120657) checkout /usr/local/lib/python3.8/dist-packages/oslo_db/sqlalchemy/engines.py:413}} Aug 18 21:47:52.799518 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writer thread running {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:99}} Aug 18 21:47:52.800208 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:47:52.841449 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=6,nova_cell0:UPDATE=3 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:47:54.075071 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-aaaa146e-bbe3-4fed-b0f5-2adbba765925 tempest-AggregatesAdminTestJSON-959991073 tempest-AggregatesAdminTestJSON-959991073-project] Took 1.26 seconds to select destinations for 1 instance(s). {{(pid=120657) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 18 21:47:54.138758 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-aaaa146e-bbe3-4fed-b0f5-2adbba765925 tempest-AggregatesAdminTestJSON-959991073 tempest-AggregatesAdminTestJSON-959991073-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 21:47:54.139534 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-aaaa146e-bbe3-4fed-b0f5-2adbba765925 tempest-AggregatesAdminTestJSON-959991073 tempest-AggregatesAdminTestJSON-959991073-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 21:47:54.142596 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-aaaa146e-bbe3-4fed-b0f5-2adbba765925 tempest-AggregatesAdminTestJSON-959991073 tempest-AggregatesAdminTestJSON-959991073-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.003s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 21:47:54.145979 n-d-853704-1 nova-conductor[120657]: INFO dbcounter [None req-aaaa146e-bbe3-4fed-b0f5-2adbba765925 tempest-AggregatesAdminTestJSON-959991073 tempest-AggregatesAdminTestJSON-959991073-project] Registered counter for database nova_cell1 Aug 18 21:47:54.154114 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_db.sqlalchemy.engines [None req-aaaa146e-bbe3-4fed-b0f5-2adbba765925 tempest-AggregatesAdminTestJSON-959991073 tempest-AggregatesAdminTestJSON-959991073-project] MySQL server mode set to STRICT_TRANS_TABLES,STRICT_ALL_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,TRADITIONAL,NO_ENGINE_SUBSTITUTION {{(pid=120657) _check_effective_sql_mode /usr/local/lib/python3.8/dist-packages/oslo_db/sqlalchemy/engines.py:314}} Aug 18 21:47:54.161408 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writer thread running {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:99}} Aug 18 21:47:54.577174 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-aaaa146e-bbe3-4fed-b0f5-2adbba765925 tempest-AggregatesAdminTestJSON-959991073 tempest-AggregatesAdminTestJSON-959991073-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 21:47:54.578274 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-aaaa146e-bbe3-4fed-b0f5-2adbba765925 tempest-AggregatesAdminTestJSON-959991073 tempest-AggregatesAdminTestJSON-959991073-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 21:47:54.582025 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-aaaa146e-bbe3-4fed-b0f5-2adbba765925 tempest-AggregatesAdminTestJSON-959991073 tempest-AggregatesAdminTestJSON-959991073-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.003s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 21:47:54.582879 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-aaaa146e-bbe3-4fed-b0f5-2adbba765925 tempest-AggregatesAdminTestJSON-959991073 tempest-AggregatesAdminTestJSON-959991073-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 21:47:54.586538 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-aaaa146e-bbe3-4fed-b0f5-2adbba765925 tempest-AggregatesAdminTestJSON-959991073 tempest-AggregatesAdminTestJSON-959991073-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 21:47:54.586538 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-aaaa146e-bbe3-4fed-b0f5-2adbba765925 tempest-AggregatesAdminTestJSON-959991073 tempest-AggregatesAdminTestJSON-959991073-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 21:47:54.587959 n-d-853704-1 nova-conductor[120657]: INFO dbcounter [None req-aaaa146e-bbe3-4fed-b0f5-2adbba765925 tempest-AggregatesAdminTestJSON-959991073 tempest-AggregatesAdminTestJSON-959991073-project] Registered counter for database nova_cell0 Aug 18 21:47:54.609252 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_db.sqlalchemy.engines [None req-aaaa146e-bbe3-4fed-b0f5-2adbba765925 tempest-AggregatesAdminTestJSON-959991073 tempest-AggregatesAdminTestJSON-959991073-project] MySQL server mode set to STRICT_TRANS_TABLES,STRICT_ALL_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,TRADITIONAL,NO_ENGINE_SUBSTITUTION {{(pid=120657) _check_effective_sql_mode /usr/local/lib/python3.8/dist-packages/oslo_db/sqlalchemy/engines.py:314}} Aug 18 21:47:54.615477 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writer thread running {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:99}} Aug 18 21:47:54.655376 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-aaaa146e-bbe3-4fed-b0f5-2adbba765925 tempest-AggregatesAdminTestJSON-959991073 tempest-AggregatesAdminTestJSON-959991073-project] Getting quotas for project f8868c787b3943c58c8b053009e2dcd9. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 18 21:47:54.668181 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-aaaa146e-bbe3-4fed-b0f5-2adbba765925 tempest-AggregatesAdminTestJSON-959991073 tempest-AggregatesAdminTestJSON-959991073-project] Getting quotas for user ee63065d37d442b4a6a90a7ff484d575 and project f8868c787b3943c58c8b053009e2dcd9. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 18 21:47:54.702962 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-aaaa146e-bbe3-4fed-b0f5-2adbba765925 tempest-AggregatesAdminTestJSON-959991073 tempest-AggregatesAdminTestJSON-959991073-project] [instance: 8882c458-572b-41bd-8cf4-28af6b377084] Selected host: n-h1-853704-1; Selected node: n-h1-853704-1; Alternates: [] {{(pid=120657) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 18 21:47:54.705083 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-aaaa146e-bbe3-4fed-b0f5-2adbba765925 tempest-AggregatesAdminTestJSON-959991073 tempest-AggregatesAdminTestJSON-959991073-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 21:47:54.705544 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-aaaa146e-bbe3-4fed-b0f5-2adbba765925 tempest-AggregatesAdminTestJSON-959991073 tempest-AggregatesAdminTestJSON-959991073-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.002s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 21:47:54.705941 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-aaaa146e-bbe3-4fed-b0f5-2adbba765925 tempest-AggregatesAdminTestJSON-959991073 tempest-AggregatesAdminTestJSON-959991073-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 21:47:54.779723 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-aaaa146e-bbe3-4fed-b0f5-2adbba765925 tempest-AggregatesAdminTestJSON-959991073 tempest-AggregatesAdminTestJSON-959991073-project] [instance: 8882c458-572b-41bd-8cf4-28af6b377084] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='87b0fec9-a948-41f9-a3c0-f0a6a5a8f1e7',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120657) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 18 21:47:54.782918 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-aaaa146e-bbe3-4fed-b0f5-2adbba765925 tempest-AggregatesAdminTestJSON-959991073 tempest-AggregatesAdminTestJSON-959991073-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 21:47:54.783552 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-aaaa146e-bbe3-4fed-b0f5-2adbba765925 tempest-AggregatesAdminTestJSON-959991073 tempest-AggregatesAdminTestJSON-959991073-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 21:47:54.784561 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-aaaa146e-bbe3-4fed-b0f5-2adbba765925 tempest-AggregatesAdminTestJSON-959991073 tempest-AggregatesAdminTestJSON-959991073-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 21:47:55.155201 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-aaaa146e-bbe3-4fed-b0f5-2adbba765925 tempest-AggregatesAdminTestJSON-959991073 tempest-AggregatesAdminTestJSON-959991073-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 21:47:55.155201 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-aaaa146e-bbe3-4fed-b0f5-2adbba765925 tempest-AggregatesAdminTestJSON-959991073 tempest-AggregatesAdminTestJSON-959991073-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 21:47:55.155201 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-aaaa146e-bbe3-4fed-b0f5-2adbba765925 tempest-AggregatesAdminTestJSON-959991073 tempest-AggregatesAdminTestJSON-959991073-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 21:47:55.155201 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-aaaa146e-bbe3-4fed-b0f5-2adbba765925 tempest-AggregatesAdminTestJSON-959991073 tempest-AggregatesAdminTestJSON-959991073-project] Acquiring lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 18 21:47:55.156071 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-aaaa146e-bbe3-4fed-b0f5-2adbba765925 tempest-AggregatesAdminTestJSON-959991073 tempest-AggregatesAdminTestJSON-959991073-project] Acquired lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 18 21:47:55.156071 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-6c40b018-c388-445a-8996-d0f667c04895 None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 21:47:55.156071 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-6c40b018-c388-445a-8996-d0f667c04895 None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 21:47:55.156071 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-6c40b018-c388-445a-8996-d0f667c04895 None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 21:47:55.156071 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-6c40b018-c388-445a-8996-d0f667c04895 None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 21:47:55.156382 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-6c40b018-c388-445a-8996-d0f667c04895 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 21:47:55.156605 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-6c40b018-c388-445a-8996-d0f667c04895 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 21:47:55.189713 n-d-853704-1 nova-conductor[120657]: ERROR nova.compute.rpcapi [None req-6c40b018-c388-445a-8996-d0f667c04895 None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 18 21:47:55.194303 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-6c40b018-c388-445a-8996-d0f667c04895 None None] Releasing lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 18 21:47:55.203095 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server [None req-6c40b018-c388-445a-8996-d0f667c04895 None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 21:47:55.203095 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 21:47:55.203095 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 18 21:47:55.203095 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 18 21:47:55.203095 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 18 21:47:55.203095 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 21:47:55.203095 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 18 21:47:55.203095 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 21:47:55.203095 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 21:47:55.203095 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 18 21:47:55.203095 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 18 21:47:55.203095 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 18 21:47:55.203095 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 18 21:47:55.203095 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 18 21:47:55.203095 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 18 21:47:55.203095 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 18 21:47:55.203095 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 18 21:47:55.204079 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 18 21:47:55.204079 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 18 21:47:55.204079 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 18 21:47:55.204079 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 18 21:47:55.204079 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 18 21:47:55.204079 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 18 21:47:55.204079 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 21:47:55.204079 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 21:48:04.947454 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=13 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:48:05.228692 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=10,nova_api:UPDATE=1,nova_api:DELETE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:48:05.228692 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=5 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:48:05.228692 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell1:SELECT=15,nova_cell1:SAVEPOINT=1,nova_cell1:INSERT=19,nova_cell1:RELEASE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:48:12.876635 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:48:21.090559 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_db.sqlalchemy.engines [None req-40d41ff6-5401-4dc3-8f3f-673ca5861a41 tempest-LiveMigrationTest-1883958412 tempest-LiveMigrationTest-1883958412-project] Parent process 119706 forked (120658) with an open database connection, which is being discarded and recreated. {{(pid=120658) checkout /usr/local/lib/python3.8/dist-packages/oslo_db/sqlalchemy/engines.py:413}} Aug 18 21:48:21.096549 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writer thread running {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:99}} Aug 18 21:48:22.078201 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-40d41ff6-5401-4dc3-8f3f-673ca5861a41 tempest-LiveMigrationTest-1883958412 tempest-LiveMigrationTest-1883958412-project] Took 0.97 seconds to select destinations for 1 instance(s). {{(pid=120658) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 18 21:48:22.171774 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:48:22.237568 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-40d41ff6-5401-4dc3-8f3f-673ca5861a41 tempest-LiveMigrationTest-1883958412 tempest-LiveMigrationTest-1883958412-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 21:48:22.238188 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-40d41ff6-5401-4dc3-8f3f-673ca5861a41 tempest-LiveMigrationTest-1883958412 tempest-LiveMigrationTest-1883958412-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 21:48:22.241816 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-40d41ff6-5401-4dc3-8f3f-673ca5861a41 tempest-LiveMigrationTest-1883958412 tempest-LiveMigrationTest-1883958412-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.004s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 21:48:22.245188 n-d-853704-1 nova-conductor[120658]: INFO dbcounter [None req-40d41ff6-5401-4dc3-8f3f-673ca5861a41 tempest-LiveMigrationTest-1883958412 tempest-LiveMigrationTest-1883958412-project] Registered counter for database nova_cell1 Aug 18 21:48:22.253649 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_db.sqlalchemy.engines [None req-40d41ff6-5401-4dc3-8f3f-673ca5861a41 tempest-LiveMigrationTest-1883958412 tempest-LiveMigrationTest-1883958412-project] MySQL server mode set to STRICT_TRANS_TABLES,STRICT_ALL_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,TRADITIONAL,NO_ENGINE_SUBSTITUTION {{(pid=120658) _check_effective_sql_mode /usr/local/lib/python3.8/dist-packages/oslo_db/sqlalchemy/engines.py:314}} Aug 18 21:48:22.260079 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writer thread running {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:99}} Aug 18 21:48:22.559222 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-40d41ff6-5401-4dc3-8f3f-673ca5861a41 tempest-LiveMigrationTest-1883958412 tempest-LiveMigrationTest-1883958412-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 21:48:22.560170 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-40d41ff6-5401-4dc3-8f3f-673ca5861a41 tempest-LiveMigrationTest-1883958412 tempest-LiveMigrationTest-1883958412-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.003s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 21:48:22.563069 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-40d41ff6-5401-4dc3-8f3f-673ca5861a41 tempest-LiveMigrationTest-1883958412 tempest-LiveMigrationTest-1883958412-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.003s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 21:48:22.564001 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-40d41ff6-5401-4dc3-8f3f-673ca5861a41 tempest-LiveMigrationTest-1883958412 tempest-LiveMigrationTest-1883958412-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 21:48:22.565590 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-40d41ff6-5401-4dc3-8f3f-673ca5861a41 tempest-LiveMigrationTest-1883958412 tempest-LiveMigrationTest-1883958412-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.002s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 21:48:22.565989 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-40d41ff6-5401-4dc3-8f3f-673ca5861a41 tempest-LiveMigrationTest-1883958412 tempest-LiveMigrationTest-1883958412-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 21:48:22.572993 n-d-853704-1 nova-conductor[120658]: INFO dbcounter [None req-40d41ff6-5401-4dc3-8f3f-673ca5861a41 tempest-LiveMigrationTest-1883958412 tempest-LiveMigrationTest-1883958412-project] Registered counter for database nova_cell0 Aug 18 21:48:22.614146 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_db.sqlalchemy.engines [None req-40d41ff6-5401-4dc3-8f3f-673ca5861a41 tempest-LiveMigrationTest-1883958412 tempest-LiveMigrationTest-1883958412-project] MySQL server mode set to STRICT_TRANS_TABLES,STRICT_ALL_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,TRADITIONAL,NO_ENGINE_SUBSTITUTION {{(pid=120658) _check_effective_sql_mode /usr/local/lib/python3.8/dist-packages/oslo_db/sqlalchemy/engines.py:314}} Aug 18 21:48:22.626004 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writer thread running {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:99}} Aug 18 21:48:22.679203 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-40d41ff6-5401-4dc3-8f3f-673ca5861a41 tempest-LiveMigrationTest-1883958412 tempest-LiveMigrationTest-1883958412-project] Getting quotas for project 29de488917ec43a09800ec6cb2187b56. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 18 21:48:22.696182 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-40d41ff6-5401-4dc3-8f3f-673ca5861a41 tempest-LiveMigrationTest-1883958412 tempest-LiveMigrationTest-1883958412-project] Getting quotas for user c137b718cd2445909cacb17658bff8f6 and project 29de488917ec43a09800ec6cb2187b56. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 18 21:48:22.735457 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-40d41ff6-5401-4dc3-8f3f-673ca5861a41 tempest-LiveMigrationTest-1883958412 tempest-LiveMigrationTest-1883958412-project] [instance: 8bebb4ec-7b93-491c-a0b4-dd5fe9778f88] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120658) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 18 21:48:22.739449 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-40d41ff6-5401-4dc3-8f3f-673ca5861a41 tempest-LiveMigrationTest-1883958412 tempest-LiveMigrationTest-1883958412-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 21:48:22.742901 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-40d41ff6-5401-4dc3-8f3f-673ca5861a41 tempest-LiveMigrationTest-1883958412 tempest-LiveMigrationTest-1883958412-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.002s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 21:48:22.743608 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-40d41ff6-5401-4dc3-8f3f-673ca5861a41 tempest-LiveMigrationTest-1883958412 tempest-LiveMigrationTest-1883958412-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.003s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 21:48:22.823118 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-40d41ff6-5401-4dc3-8f3f-673ca5861a41 tempest-LiveMigrationTest-1883958412 tempest-LiveMigrationTest-1883958412-project] [instance: 8bebb4ec-7b93-491c-a0b4-dd5fe9778f88] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='87b0fec9-a948-41f9-a3c0-f0a6a5a8f1e7',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120658) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 18 21:48:22.827002 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-40d41ff6-5401-4dc3-8f3f-673ca5861a41 tempest-LiveMigrationTest-1883958412 tempest-LiveMigrationTest-1883958412-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 21:48:22.827604 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-40d41ff6-5401-4dc3-8f3f-673ca5861a41 tempest-LiveMigrationTest-1883958412 tempest-LiveMigrationTest-1883958412-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 21:48:22.828166 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-40d41ff6-5401-4dc3-8f3f-673ca5861a41 tempest-LiveMigrationTest-1883958412 tempest-LiveMigrationTest-1883958412-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 21:48:23.156472 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-40d41ff6-5401-4dc3-8f3f-673ca5861a41 tempest-LiveMigrationTest-1883958412 tempest-LiveMigrationTest-1883958412-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 21:48:23.157079 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-40d41ff6-5401-4dc3-8f3f-673ca5861a41 tempest-LiveMigrationTest-1883958412 tempest-LiveMigrationTest-1883958412-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 21:48:23.157799 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-40d41ff6-5401-4dc3-8f3f-673ca5861a41 tempest-LiveMigrationTest-1883958412 tempest-LiveMigrationTest-1883958412-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 21:48:23.158528 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-40d41ff6-5401-4dc3-8f3f-673ca5861a41 tempest-LiveMigrationTest-1883958412 tempest-LiveMigrationTest-1883958412-project] Acquiring lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 18 21:48:23.161355 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-40d41ff6-5401-4dc3-8f3f-673ca5861a41 tempest-LiveMigrationTest-1883958412 tempest-LiveMigrationTest-1883958412-project] Acquired lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 18 21:48:23.163095 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-addb9f50-3d96-4a7c-9e0d-d7922ba32c18 None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 21:48:23.163774 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-addb9f50-3d96-4a7c-9e0d-d7922ba32c18 None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 21:48:23.165606 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-addb9f50-3d96-4a7c-9e0d-d7922ba32c18 None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 21:48:23.166501 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-addb9f50-3d96-4a7c-9e0d-d7922ba32c18 None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 21:48:23.168301 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-addb9f50-3d96-4a7c-9e0d-d7922ba32c18 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 21:48:23.168904 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-addb9f50-3d96-4a7c-9e0d-d7922ba32c18 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 21:48:23.202421 n-d-853704-1 nova-conductor[120658]: ERROR nova.compute.rpcapi [None req-addb9f50-3d96-4a7c-9e0d-d7922ba32c18 None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 18 21:48:23.204076 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-addb9f50-3d96-4a7c-9e0d-d7922ba32c18 None None] Releasing lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 18 21:48:23.226002 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server [None req-addb9f50-3d96-4a7c-9e0d-d7922ba32c18 None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 21:48:23.226002 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 21:48:23.226002 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 18 21:48:23.226002 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 18 21:48:23.226002 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 18 21:48:23.226002 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 21:48:23.226002 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 18 21:48:23.226002 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 21:48:23.226002 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 21:48:23.226002 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 18 21:48:23.226002 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 18 21:48:23.226002 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 18 21:48:23.226002 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 18 21:48:23.226002 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 18 21:48:23.226002 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 18 21:48:23.226002 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 18 21:48:23.226002 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 18 21:48:23.229882 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 18 21:48:23.229882 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 18 21:48:23.229882 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 18 21:48:23.229882 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 18 21:48:23.229882 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 18 21:48:23.229882 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 18 21:48:23.229882 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 21:48:23.229882 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 21:48:23.465622 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-8a1d1123-798a-4953-a6d0-a85b972e0693 tempest-ServersWithSpecificFlavorTestJSON-1230393595 tempest-ServersWithSpecificFlavorTestJSON-1230393595-project] Took 1.32 seconds to select destinations for 1 instance(s). {{(pid=120657) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 18 21:48:23.489808 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-8a1d1123-798a-4953-a6d0-a85b972e0693 tempest-ServersWithSpecificFlavorTestJSON-1230393595 tempest-ServersWithSpecificFlavorTestJSON-1230393595-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 21:48:23.490465 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-8a1d1123-798a-4953-a6d0-a85b972e0693 tempest-ServersWithSpecificFlavorTestJSON-1230393595 tempest-ServersWithSpecificFlavorTestJSON-1230393595-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 21:48:23.491775 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-8a1d1123-798a-4953-a6d0-a85b972e0693 tempest-ServersWithSpecificFlavorTestJSON-1230393595 tempest-ServersWithSpecificFlavorTestJSON-1230393595-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 21:48:23.796349 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=15,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:48:23.819650 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-8a1d1123-798a-4953-a6d0-a85b972e0693 tempest-ServersWithSpecificFlavorTestJSON-1230393595 tempest-ServersWithSpecificFlavorTestJSON-1230393595-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 21:48:23.821341 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-8a1d1123-798a-4953-a6d0-a85b972e0693 tempest-ServersWithSpecificFlavorTestJSON-1230393595 tempest-ServersWithSpecificFlavorTestJSON-1230393595-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.002s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 21:48:23.823171 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-8a1d1123-798a-4953-a6d0-a85b972e0693 tempest-ServersWithSpecificFlavorTestJSON-1230393595 tempest-ServersWithSpecificFlavorTestJSON-1230393595-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.002s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 21:48:23.825771 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-8a1d1123-798a-4953-a6d0-a85b972e0693 tempest-ServersWithSpecificFlavorTestJSON-1230393595 tempest-ServersWithSpecificFlavorTestJSON-1230393595-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 21:48:23.832530 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-8a1d1123-798a-4953-a6d0-a85b972e0693 tempest-ServersWithSpecificFlavorTestJSON-1230393595 tempest-ServersWithSpecificFlavorTestJSON-1230393595-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.005s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 21:48:23.832530 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-8a1d1123-798a-4953-a6d0-a85b972e0693 tempest-ServersWithSpecificFlavorTestJSON-1230393595 tempest-ServersWithSpecificFlavorTestJSON-1230393595-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 21:48:23.877333 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-8a1d1123-798a-4953-a6d0-a85b972e0693 tempest-ServersWithSpecificFlavorTestJSON-1230393595 tempest-ServersWithSpecificFlavorTestJSON-1230393595-project] Getting quotas for project 5b1c0dc86d3a495298fa08c92b75c36d. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 18 21:48:23.881994 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-8a1d1123-798a-4953-a6d0-a85b972e0693 tempest-ServersWithSpecificFlavorTestJSON-1230393595 tempest-ServersWithSpecificFlavorTestJSON-1230393595-project] Getting quotas for user 68694bea7ef74991adfd14c04826783d and project 5b1c0dc86d3a495298fa08c92b75c36d. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 18 21:48:23.898575 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-8a1d1123-798a-4953-a6d0-a85b972e0693 tempest-ServersWithSpecificFlavorTestJSON-1230393595 tempest-ServersWithSpecificFlavorTestJSON-1230393595-project] [instance: 064c117b-ed56-4aa5-b38f-c7dafc5bf1f8] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120657) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 18 21:48:23.899585 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-8a1d1123-798a-4953-a6d0-a85b972e0693 tempest-ServersWithSpecificFlavorTestJSON-1230393595 tempest-ServersWithSpecificFlavorTestJSON-1230393595-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 21:48:23.900054 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-8a1d1123-798a-4953-a6d0-a85b972e0693 tempest-ServersWithSpecificFlavorTestJSON-1230393595 tempest-ServersWithSpecificFlavorTestJSON-1230393595-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 21:48:23.900447 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-8a1d1123-798a-4953-a6d0-a85b972e0693 tempest-ServersWithSpecificFlavorTestJSON-1230393595 tempest-ServersWithSpecificFlavorTestJSON-1230393595-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 21:48:24.005540 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-8a1d1123-798a-4953-a6d0-a85b972e0693 tempest-ServersWithSpecificFlavorTestJSON-1230393595 tempest-ServersWithSpecificFlavorTestJSON-1230393595-project] [instance: 064c117b-ed56-4aa5-b38f-c7dafc5bf1f8] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='87b0fec9-a948-41f9-a3c0-f0a6a5a8f1e7',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120657) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 18 21:48:24.007081 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-8a1d1123-798a-4953-a6d0-a85b972e0693 tempest-ServersWithSpecificFlavorTestJSON-1230393595 tempest-ServersWithSpecificFlavorTestJSON-1230393595-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 21:48:24.007609 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-8a1d1123-798a-4953-a6d0-a85b972e0693 tempest-ServersWithSpecificFlavorTestJSON-1230393595 tempest-ServersWithSpecificFlavorTestJSON-1230393595-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 21:48:24.008254 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-8a1d1123-798a-4953-a6d0-a85b972e0693 tempest-ServersWithSpecificFlavorTestJSON-1230393595 tempest-ServersWithSpecificFlavorTestJSON-1230393595-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 21:48:24.292282 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-8a1d1123-798a-4953-a6d0-a85b972e0693 tempest-ServersWithSpecificFlavorTestJSON-1230393595 tempest-ServersWithSpecificFlavorTestJSON-1230393595-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 21:48:24.293043 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-8a1d1123-798a-4953-a6d0-a85b972e0693 tempest-ServersWithSpecificFlavorTestJSON-1230393595 tempest-ServersWithSpecificFlavorTestJSON-1230393595-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 21:48:24.294025 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-8a1d1123-798a-4953-a6d0-a85b972e0693 tempest-ServersWithSpecificFlavorTestJSON-1230393595 tempest-ServersWithSpecificFlavorTestJSON-1230393595-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 21:48:24.294568 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-8a1d1123-798a-4953-a6d0-a85b972e0693 tempest-ServersWithSpecificFlavorTestJSON-1230393595 tempest-ServersWithSpecificFlavorTestJSON-1230393595-project] Acquiring lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 18 21:48:24.295000 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-8a1d1123-798a-4953-a6d0-a85b972e0693 tempest-ServersWithSpecificFlavorTestJSON-1230393595 tempest-ServersWithSpecificFlavorTestJSON-1230393595-project] Acquired lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 18 21:48:24.295865 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-65878bf5-b12a-440a-a478-8c3f9d9237a4 None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 21:48:24.296339 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-65878bf5-b12a-440a-a478-8c3f9d9237a4 None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 21:48:24.298081 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-65878bf5-b12a-440a-a478-8c3f9d9237a4 None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 21:48:24.298081 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-65878bf5-b12a-440a-a478-8c3f9d9237a4 None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 21:48:24.298081 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-65878bf5-b12a-440a-a478-8c3f9d9237a4 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 21:48:24.299049 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-65878bf5-b12a-440a-a478-8c3f9d9237a4 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 21:48:24.358896 n-d-853704-1 nova-conductor[120657]: ERROR nova.compute.rpcapi [None req-65878bf5-b12a-440a-a478-8c3f9d9237a4 None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 18 21:48:24.360029 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-65878bf5-b12a-440a-a478-8c3f9d9237a4 None None] Releasing lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 18 21:48:24.361037 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server [None req-65878bf5-b12a-440a-a478-8c3f9d9237a4 None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 21:48:24.361037 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 21:48:24.361037 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 18 21:48:24.361037 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 18 21:48:24.361037 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 18 21:48:24.361037 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 21:48:24.361037 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 18 21:48:24.361037 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 21:48:24.361037 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 21:48:24.361037 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 18 21:48:24.361037 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 18 21:48:24.361037 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 18 21:48:24.361037 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 18 21:48:24.361037 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 18 21:48:24.361037 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 18 21:48:24.361037 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 18 21:48:24.361037 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 18 21:48:24.362228 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 18 21:48:24.362228 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 18 21:48:24.362228 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 18 21:48:24.362228 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 18 21:48:24.362228 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 18 21:48:24.362228 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 18 21:48:24.362228 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 21:48:24.362228 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 21:48:32.891973 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=2 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:48:32.969181 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=7,nova_api:UPDATE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:48:33.066334 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=16,nova_api:DELETE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:48:33.189657 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=5 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:48:33.190495 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell1:SELECT=12,nova_cell1:SAVEPOINT=1,nova_cell1:INSERT=18,nova_cell1:RELEASE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:48:34.145118 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=12,nova_api:UPDATE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:48:34.216335 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=12,nova_api:DELETE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:48:34.350259 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:48:34.351306 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell1:SELECT=11,nova_cell1:SAVEPOINT=1,nova_cell1:INSERT=18,nova_cell1:RELEASE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:48:40.730415 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-d6c18921-9b68-4983-b2ae-576d85798383 tempest-ServersAdminNegativeTestJSON-1990898318 tempest-ServersAdminNegativeTestJSON-1990898318-project] Took 0.74 seconds to select destinations for 1 instance(s). {{(pid=120658) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 18 21:48:40.760486 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-d6c18921-9b68-4983-b2ae-576d85798383 tempest-ServersAdminNegativeTestJSON-1990898318 tempest-ServersAdminNegativeTestJSON-1990898318-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 21:48:40.761337 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-d6c18921-9b68-4983-b2ae-576d85798383 tempest-ServersAdminNegativeTestJSON-1990898318 tempest-ServersAdminNegativeTestJSON-1990898318-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 21:48:40.762004 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-d6c18921-9b68-4983-b2ae-576d85798383 tempest-ServersAdminNegativeTestJSON-1990898318 tempest-ServersAdminNegativeTestJSON-1990898318-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 21:48:40.992192 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-d6c18921-9b68-4983-b2ae-576d85798383 tempest-ServersAdminNegativeTestJSON-1990898318 tempest-ServersAdminNegativeTestJSON-1990898318-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 21:48:40.992667 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-d6c18921-9b68-4983-b2ae-576d85798383 tempest-ServersAdminNegativeTestJSON-1990898318 tempest-ServersAdminNegativeTestJSON-1990898318-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 21:48:40.993100 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-d6c18921-9b68-4983-b2ae-576d85798383 tempest-ServersAdminNegativeTestJSON-1990898318 tempest-ServersAdminNegativeTestJSON-1990898318-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 21:48:40.994388 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-d6c18921-9b68-4983-b2ae-576d85798383 tempest-ServersAdminNegativeTestJSON-1990898318 tempest-ServersAdminNegativeTestJSON-1990898318-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 21:48:40.995176 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-d6c18921-9b68-4983-b2ae-576d85798383 tempest-ServersAdminNegativeTestJSON-1990898318 tempest-ServersAdminNegativeTestJSON-1990898318-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 21:48:40.995991 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-d6c18921-9b68-4983-b2ae-576d85798383 tempest-ServersAdminNegativeTestJSON-1990898318 tempest-ServersAdminNegativeTestJSON-1990898318-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 21:48:41.018404 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-d6c18921-9b68-4983-b2ae-576d85798383 tempest-ServersAdminNegativeTestJSON-1990898318 tempest-ServersAdminNegativeTestJSON-1990898318-project] Getting quotas for project 9575c5869ef4458796e8447f19888268. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 18 21:48:41.023479 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-d6c18921-9b68-4983-b2ae-576d85798383 tempest-ServersAdminNegativeTestJSON-1990898318 tempest-ServersAdminNegativeTestJSON-1990898318-project] Getting quotas for user a51c589119af4ad2abb9c4af04fd68f6 and project 9575c5869ef4458796e8447f19888268. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 18 21:48:41.034981 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-d6c18921-9b68-4983-b2ae-576d85798383 tempest-ServersAdminNegativeTestJSON-1990898318 tempest-ServersAdminNegativeTestJSON-1990898318-project] [instance: 3f4b9624-a411-417d-ada0-3558dce55cb9] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120658) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 18 21:48:41.036123 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-d6c18921-9b68-4983-b2ae-576d85798383 tempest-ServersAdminNegativeTestJSON-1990898318 tempest-ServersAdminNegativeTestJSON-1990898318-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 21:48:41.036715 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-d6c18921-9b68-4983-b2ae-576d85798383 tempest-ServersAdminNegativeTestJSON-1990898318 tempest-ServersAdminNegativeTestJSON-1990898318-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 21:48:41.037284 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-d6c18921-9b68-4983-b2ae-576d85798383 tempest-ServersAdminNegativeTestJSON-1990898318 tempest-ServersAdminNegativeTestJSON-1990898318-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 21:48:41.100304 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-d6c18921-9b68-4983-b2ae-576d85798383 tempest-ServersAdminNegativeTestJSON-1990898318 tempest-ServersAdminNegativeTestJSON-1990898318-project] [instance: 3f4b9624-a411-417d-ada0-3558dce55cb9] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='87b0fec9-a948-41f9-a3c0-f0a6a5a8f1e7',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120658) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 18 21:48:41.108742 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-d6c18921-9b68-4983-b2ae-576d85798383 tempest-ServersAdminNegativeTestJSON-1990898318 tempest-ServersAdminNegativeTestJSON-1990898318-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 21:48:41.110565 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-d6c18921-9b68-4983-b2ae-576d85798383 tempest-ServersAdminNegativeTestJSON-1990898318 tempest-ServersAdminNegativeTestJSON-1990898318-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.002s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 21:48:41.111239 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-d6c18921-9b68-4983-b2ae-576d85798383 tempest-ServersAdminNegativeTestJSON-1990898318 tempest-ServersAdminNegativeTestJSON-1990898318-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 21:48:41.345779 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-d6c18921-9b68-4983-b2ae-576d85798383 tempest-ServersAdminNegativeTestJSON-1990898318 tempest-ServersAdminNegativeTestJSON-1990898318-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 21:48:41.346335 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-d6c18921-9b68-4983-b2ae-576d85798383 tempest-ServersAdminNegativeTestJSON-1990898318 tempest-ServersAdminNegativeTestJSON-1990898318-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 21:48:41.346889 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-d6c18921-9b68-4983-b2ae-576d85798383 tempest-ServersAdminNegativeTestJSON-1990898318 tempest-ServersAdminNegativeTestJSON-1990898318-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 21:48:41.347525 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-d6c18921-9b68-4983-b2ae-576d85798383 tempest-ServersAdminNegativeTestJSON-1990898318 tempest-ServersAdminNegativeTestJSON-1990898318-project] Acquiring lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 18 21:48:41.347989 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-d6c18921-9b68-4983-b2ae-576d85798383 tempest-ServersAdminNegativeTestJSON-1990898318 tempest-ServersAdminNegativeTestJSON-1990898318-project] Acquired lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 18 21:48:41.348753 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-5870aa28-63b2-4147-b16d-d5614e32fbfc None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 21:48:41.349173 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-5870aa28-63b2-4147-b16d-d5614e32fbfc None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 21:48:41.349559 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-5870aa28-63b2-4147-b16d-d5614e32fbfc None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 21:48:41.350229 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-5870aa28-63b2-4147-b16d-d5614e32fbfc None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 21:48:41.352959 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-5870aa28-63b2-4147-b16d-d5614e32fbfc None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 21:48:41.352959 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-5870aa28-63b2-4147-b16d-d5614e32fbfc None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 21:48:41.361789 n-d-853704-1 nova-conductor[120658]: ERROR nova.compute.rpcapi [None req-5870aa28-63b2-4147-b16d-d5614e32fbfc None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 18 21:48:41.362634 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-5870aa28-63b2-4147-b16d-d5614e32fbfc None None] Releasing lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 18 21:48:41.363543 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server [None req-5870aa28-63b2-4147-b16d-d5614e32fbfc None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 21:48:41.363543 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 21:48:41.363543 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 18 21:48:41.363543 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 18 21:48:41.363543 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 18 21:48:41.363543 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 21:48:41.363543 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 18 21:48:41.363543 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 21:48:41.363543 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 21:48:41.363543 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 18 21:48:41.363543 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 18 21:48:41.363543 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 18 21:48:41.363543 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 18 21:48:41.363543 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 18 21:48:41.363543 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 18 21:48:41.363543 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 18 21:48:41.363543 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 18 21:48:41.364606 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 18 21:48:41.364606 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 18 21:48:41.364606 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 18 21:48:41.364606 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 18 21:48:41.364606 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 18 21:48:41.364606 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 18 21:48:41.364606 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 21:48:41.364606 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 21:48:51.080083 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-a2e715fb-96cb-45c1-b61e-518f834a8e54 tempest-LiveAutoBlockMigrationV225Test-1869140755 tempest-LiveAutoBlockMigrationV225Test-1869140755-project] Took 0.70 seconds to select destinations for 1 instance(s). {{(pid=120657) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 18 21:48:51.101448 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-a2e715fb-96cb-45c1-b61e-518f834a8e54 tempest-LiveAutoBlockMigrationV225Test-1869140755 tempest-LiveAutoBlockMigrationV225Test-1869140755-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 21:48:51.101890 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-a2e715fb-96cb-45c1-b61e-518f834a8e54 tempest-LiveAutoBlockMigrationV225Test-1869140755 tempest-LiveAutoBlockMigrationV225Test-1869140755-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 21:48:51.102297 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-a2e715fb-96cb-45c1-b61e-518f834a8e54 tempest-LiveAutoBlockMigrationV225Test-1869140755 tempest-LiveAutoBlockMigrationV225Test-1869140755-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 21:48:51.189375 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=13,nova_api:UPDATE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:48:51.252599 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=11,nova_api:DELETE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:48:51.306976 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-a2e715fb-96cb-45c1-b61e-518f834a8e54 tempest-LiveAutoBlockMigrationV225Test-1869140755 tempest-LiveAutoBlockMigrationV225Test-1869140755-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 21:48:51.308092 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-a2e715fb-96cb-45c1-b61e-518f834a8e54 tempest-LiveAutoBlockMigrationV225Test-1869140755 tempest-LiveAutoBlockMigrationV225Test-1869140755-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 21:48:51.308777 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-a2e715fb-96cb-45c1-b61e-518f834a8e54 tempest-LiveAutoBlockMigrationV225Test-1869140755 tempest-LiveAutoBlockMigrationV225Test-1869140755-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 21:48:51.309276 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-a2e715fb-96cb-45c1-b61e-518f834a8e54 tempest-LiveAutoBlockMigrationV225Test-1869140755 tempest-LiveAutoBlockMigrationV225Test-1869140755-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 21:48:51.309800 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-a2e715fb-96cb-45c1-b61e-518f834a8e54 tempest-LiveAutoBlockMigrationV225Test-1869140755 tempest-LiveAutoBlockMigrationV225Test-1869140755-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 21:48:51.310080 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-a2e715fb-96cb-45c1-b61e-518f834a8e54 tempest-LiveAutoBlockMigrationV225Test-1869140755 tempest-LiveAutoBlockMigrationV225Test-1869140755-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 21:48:51.327029 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-a2e715fb-96cb-45c1-b61e-518f834a8e54 tempest-LiveAutoBlockMigrationV225Test-1869140755 tempest-LiveAutoBlockMigrationV225Test-1869140755-project] Getting quotas for project 4c471d428ca241faa5d631d226526b3b. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 18 21:48:51.330563 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-a2e715fb-96cb-45c1-b61e-518f834a8e54 tempest-LiveAutoBlockMigrationV225Test-1869140755 tempest-LiveAutoBlockMigrationV225Test-1869140755-project] Getting quotas for user faee7152cc454a189fcbdd8f2d4b7b29 and project 4c471d428ca241faa5d631d226526b3b. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 18 21:48:51.341068 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-a2e715fb-96cb-45c1-b61e-518f834a8e54 tempest-LiveAutoBlockMigrationV225Test-1869140755 tempest-LiveAutoBlockMigrationV225Test-1869140755-project] [instance: b8f97ace-0c6d-4b7a-977d-ae77f441834a] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120657) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 18 21:48:51.341837 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-a2e715fb-96cb-45c1-b61e-518f834a8e54 tempest-LiveAutoBlockMigrationV225Test-1869140755 tempest-LiveAutoBlockMigrationV225Test-1869140755-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 21:48:51.342273 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-a2e715fb-96cb-45c1-b61e-518f834a8e54 tempest-LiveAutoBlockMigrationV225Test-1869140755 tempest-LiveAutoBlockMigrationV225Test-1869140755-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 21:48:51.342663 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-a2e715fb-96cb-45c1-b61e-518f834a8e54 tempest-LiveAutoBlockMigrationV225Test-1869140755 tempest-LiveAutoBlockMigrationV225Test-1869140755-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 21:48:51.356774 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:48:51.357686 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell1:SELECT=11,nova_cell1:SAVEPOINT=1,nova_cell1:INSERT=18,nova_cell1:RELEASE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:48:51.460715 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-a2e715fb-96cb-45c1-b61e-518f834a8e54 tempest-LiveAutoBlockMigrationV225Test-1869140755 tempest-LiveAutoBlockMigrationV225Test-1869140755-project] [instance: b8f97ace-0c6d-4b7a-977d-ae77f441834a] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='87b0fec9-a948-41f9-a3c0-f0a6a5a8f1e7',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120657) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 18 21:48:51.461999 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-a2e715fb-96cb-45c1-b61e-518f834a8e54 tempest-LiveAutoBlockMigrationV225Test-1869140755 tempest-LiveAutoBlockMigrationV225Test-1869140755-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 21:48:51.462460 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-a2e715fb-96cb-45c1-b61e-518f834a8e54 tempest-LiveAutoBlockMigrationV225Test-1869140755 tempest-LiveAutoBlockMigrationV225Test-1869140755-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 21:48:51.462861 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-a2e715fb-96cb-45c1-b61e-518f834a8e54 tempest-LiveAutoBlockMigrationV225Test-1869140755 tempest-LiveAutoBlockMigrationV225Test-1869140755-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 21:48:51.794272 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-a2e715fb-96cb-45c1-b61e-518f834a8e54 tempest-LiveAutoBlockMigrationV225Test-1869140755 tempest-LiveAutoBlockMigrationV225Test-1869140755-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 21:48:51.794885 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-a2e715fb-96cb-45c1-b61e-518f834a8e54 tempest-LiveAutoBlockMigrationV225Test-1869140755 tempest-LiveAutoBlockMigrationV225Test-1869140755-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 21:48:51.795579 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-a2e715fb-96cb-45c1-b61e-518f834a8e54 tempest-LiveAutoBlockMigrationV225Test-1869140755 tempest-LiveAutoBlockMigrationV225Test-1869140755-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 21:48:51.796145 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-a2e715fb-96cb-45c1-b61e-518f834a8e54 tempest-LiveAutoBlockMigrationV225Test-1869140755 tempest-LiveAutoBlockMigrationV225Test-1869140755-project] Acquiring lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 18 21:48:51.796664 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-a2e715fb-96cb-45c1-b61e-518f834a8e54 tempest-LiveAutoBlockMigrationV225Test-1869140755 tempest-LiveAutoBlockMigrationV225Test-1869140755-project] Acquired lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 18 21:48:51.797861 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-cf8a5baf-5cb2-4423-b386-11f1115ade50 None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 21:48:51.798269 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-cf8a5baf-5cb2-4423-b386-11f1115ade50 None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 21:48:51.801309 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-cf8a5baf-5cb2-4423-b386-11f1115ade50 None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 21:48:51.801309 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-cf8a5baf-5cb2-4423-b386-11f1115ade50 None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 21:48:51.801309 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-cf8a5baf-5cb2-4423-b386-11f1115ade50 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 21:48:51.801309 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-cf8a5baf-5cb2-4423-b386-11f1115ade50 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 21:48:51.817529 n-d-853704-1 nova-conductor[120657]: ERROR nova.compute.rpcapi [None req-cf8a5baf-5cb2-4423-b386-11f1115ade50 None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 18 21:48:51.817987 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-cf8a5baf-5cb2-4423-b386-11f1115ade50 None None] Releasing lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 18 21:48:51.818989 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server [None req-cf8a5baf-5cb2-4423-b386-11f1115ade50 None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 21:48:51.818989 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 21:48:51.818989 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 18 21:48:51.818989 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 18 21:48:51.818989 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 18 21:48:51.818989 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 21:48:51.818989 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 18 21:48:51.818989 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 21:48:51.818989 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 21:48:51.818989 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 18 21:48:51.818989 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 18 21:48:51.818989 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 18 21:48:51.818989 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 18 21:48:51.818989 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 18 21:48:51.818989 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 18 21:48:51.818989 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 18 21:48:51.818989 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 18 21:48:51.820182 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 18 21:48:51.820182 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 18 21:48:51.820182 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 18 21:48:51.820182 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 18 21:48:51.820182 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 18 21:48:51.820182 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 18 21:48:51.820182 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 21:48:51.820182 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 21:49:01.591733 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=13 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:49:01.703984 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=11,nova_api:UPDATE=1,nova_api:DELETE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:49:01.812724 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:49:01.814638 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell1:SELECT=11,nova_cell1:SAVEPOINT=1,nova_cell1:INSERT=18,nova_cell1:RELEASE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:49:02.845270 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=18,nova_cell0:UPDATE=3 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:49:12.877652 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=15,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:49:35.833243 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-1295f6c3-e133-420c-9a95-7bdc2bbe347d tempest-ImagesNegativeTestJSON-24732441 tempest-ImagesNegativeTestJSON-24732441-project] Took 0.72 seconds to select destinations for 1 instance(s). {{(pid=120658) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 18 21:49:35.869409 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-1295f6c3-e133-420c-9a95-7bdc2bbe347d tempest-ImagesNegativeTestJSON-24732441 tempest-ImagesNegativeTestJSON-24732441-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 21:49:35.871706 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-1295f6c3-e133-420c-9a95-7bdc2bbe347d tempest-ImagesNegativeTestJSON-24732441 tempest-ImagesNegativeTestJSON-24732441-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 21:49:35.871706 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-1295f6c3-e133-420c-9a95-7bdc2bbe347d tempest-ImagesNegativeTestJSON-24732441 tempest-ImagesNegativeTestJSON-24732441-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 21:49:36.089178 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-1295f6c3-e133-420c-9a95-7bdc2bbe347d tempest-ImagesNegativeTestJSON-24732441 tempest-ImagesNegativeTestJSON-24732441-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 21:49:36.089634 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-1295f6c3-e133-420c-9a95-7bdc2bbe347d tempest-ImagesNegativeTestJSON-24732441 tempest-ImagesNegativeTestJSON-24732441-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 21:49:36.090035 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-1295f6c3-e133-420c-9a95-7bdc2bbe347d tempest-ImagesNegativeTestJSON-24732441 tempest-ImagesNegativeTestJSON-24732441-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 21:49:36.090692 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-1295f6c3-e133-420c-9a95-7bdc2bbe347d tempest-ImagesNegativeTestJSON-24732441 tempest-ImagesNegativeTestJSON-24732441-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 21:49:36.091282 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-1295f6c3-e133-420c-9a95-7bdc2bbe347d tempest-ImagesNegativeTestJSON-24732441 tempest-ImagesNegativeTestJSON-24732441-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 21:49:36.091858 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-1295f6c3-e133-420c-9a95-7bdc2bbe347d tempest-ImagesNegativeTestJSON-24732441 tempest-ImagesNegativeTestJSON-24732441-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 21:49:36.134477 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-1295f6c3-e133-420c-9a95-7bdc2bbe347d tempest-ImagesNegativeTestJSON-24732441 tempest-ImagesNegativeTestJSON-24732441-project] Getting quotas for project c6ae990fa9e24c01b492b084336a0062. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 18 21:49:36.140397 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-1295f6c3-e133-420c-9a95-7bdc2bbe347d tempest-ImagesNegativeTestJSON-24732441 tempest-ImagesNegativeTestJSON-24732441-project] Getting quotas for user 024bb3458bbc4acf921442960dd130ff and project c6ae990fa9e24c01b492b084336a0062. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 18 21:49:36.180695 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-1295f6c3-e133-420c-9a95-7bdc2bbe347d tempest-ImagesNegativeTestJSON-24732441 tempest-ImagesNegativeTestJSON-24732441-project] [instance: 7d1bb094-670c-4c6c-8bc2-18563e1081e7] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120658) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 18 21:49:36.181798 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-1295f6c3-e133-420c-9a95-7bdc2bbe347d tempest-ImagesNegativeTestJSON-24732441 tempest-ImagesNegativeTestJSON-24732441-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 21:49:36.182400 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-1295f6c3-e133-420c-9a95-7bdc2bbe347d tempest-ImagesNegativeTestJSON-24732441 tempest-ImagesNegativeTestJSON-24732441-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 21:49:36.182950 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-1295f6c3-e133-420c-9a95-7bdc2bbe347d tempest-ImagesNegativeTestJSON-24732441 tempest-ImagesNegativeTestJSON-24732441-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 21:49:36.227933 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-1295f6c3-e133-420c-9a95-7bdc2bbe347d tempest-ImagesNegativeTestJSON-24732441 tempest-ImagesNegativeTestJSON-24732441-project] [instance: 7d1bb094-670c-4c6c-8bc2-18563e1081e7] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='87b0fec9-a948-41f9-a3c0-f0a6a5a8f1e7',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120658) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 18 21:49:36.229617 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-1295f6c3-e133-420c-9a95-7bdc2bbe347d tempest-ImagesNegativeTestJSON-24732441 tempest-ImagesNegativeTestJSON-24732441-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 21:49:36.230251 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-1295f6c3-e133-420c-9a95-7bdc2bbe347d tempest-ImagesNegativeTestJSON-24732441 tempest-ImagesNegativeTestJSON-24732441-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 21:49:36.232100 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-1295f6c3-e133-420c-9a95-7bdc2bbe347d tempest-ImagesNegativeTestJSON-24732441 tempest-ImagesNegativeTestJSON-24732441-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.002s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 21:49:36.462185 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-1295f6c3-e133-420c-9a95-7bdc2bbe347d tempest-ImagesNegativeTestJSON-24732441 tempest-ImagesNegativeTestJSON-24732441-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 21:49:36.462654 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-1295f6c3-e133-420c-9a95-7bdc2bbe347d tempest-ImagesNegativeTestJSON-24732441 tempest-ImagesNegativeTestJSON-24732441-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 21:49:36.463477 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-1295f6c3-e133-420c-9a95-7bdc2bbe347d tempest-ImagesNegativeTestJSON-24732441 tempest-ImagesNegativeTestJSON-24732441-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 21:49:36.464236 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-1295f6c3-e133-420c-9a95-7bdc2bbe347d tempest-ImagesNegativeTestJSON-24732441 tempest-ImagesNegativeTestJSON-24732441-project] Acquiring lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 18 21:49:36.465091 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-1295f6c3-e133-420c-9a95-7bdc2bbe347d tempest-ImagesNegativeTestJSON-24732441 tempest-ImagesNegativeTestJSON-24732441-project] Acquired lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 18 21:49:36.465911 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-2c5ad21f-b0dc-4581-b123-3fd37f88b7ed None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 21:49:36.466341 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-2c5ad21f-b0dc-4581-b123-3fd37f88b7ed None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 21:49:36.466778 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-2c5ad21f-b0dc-4581-b123-3fd37f88b7ed None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 21:49:36.467441 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-2c5ad21f-b0dc-4581-b123-3fd37f88b7ed None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 21:49:36.467891 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-2c5ad21f-b0dc-4581-b123-3fd37f88b7ed None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 21:49:36.468278 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-2c5ad21f-b0dc-4581-b123-3fd37f88b7ed None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 21:49:36.489818 n-d-853704-1 nova-conductor[120658]: ERROR nova.compute.rpcapi [None req-2c5ad21f-b0dc-4581-b123-3fd37f88b7ed None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 18 21:49:36.490206 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-2c5ad21f-b0dc-4581-b123-3fd37f88b7ed None None] Releasing lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 18 21:49:36.492258 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server [None req-2c5ad21f-b0dc-4581-b123-3fd37f88b7ed None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 21:49:36.492258 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 21:49:36.492258 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 18 21:49:36.492258 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 18 21:49:36.492258 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 18 21:49:36.492258 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 21:49:36.492258 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 18 21:49:36.492258 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 21:49:36.492258 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 21:49:36.492258 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 18 21:49:36.492258 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 18 21:49:36.492258 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 18 21:49:36.492258 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 18 21:49:36.492258 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 18 21:49:36.492258 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 18 21:49:36.492258 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 18 21:49:36.492258 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 18 21:49:36.493557 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 18 21:49:36.493557 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 18 21:49:36.493557 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 18 21:49:36.493557 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 18 21:49:36.493557 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 18 21:49:36.493557 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 18 21:49:36.493557 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 21:49:36.493557 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 21:49:42.843124 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:49:46.328570 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=15,nova_api:UPDATE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:49:46.413352 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=9,nova_api:DELETE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:49:46.487194 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:49:46.488409 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell1:SELECT=11,nova_cell1:SAVEPOINT=1,nova_cell1:INSERT=18,nova_cell1:RELEASE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:49:52.883799 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=13,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:50:12.890043 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=4,nova_cell0:UPDATE=2 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:50:14.131241 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-97aadf1d-efbf-4885-9c29-0d5dd74e636c tempest-ServersTestMultiNic-773389532 tempest-ServersTestMultiNic-773389532-project] Took 0.62 seconds to select destinations for 1 instance(s). {{(pid=120657) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 18 21:50:14.156471 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-97aadf1d-efbf-4885-9c29-0d5dd74e636c tempest-ServersTestMultiNic-773389532 tempest-ServersTestMultiNic-773389532-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 21:50:14.157465 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-97aadf1d-efbf-4885-9c29-0d5dd74e636c tempest-ServersTestMultiNic-773389532 tempest-ServersTestMultiNic-773389532-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 21:50:14.158715 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-97aadf1d-efbf-4885-9c29-0d5dd74e636c tempest-ServersTestMultiNic-773389532 tempest-ServersTestMultiNic-773389532-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 21:50:14.319958 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:50:14.324406 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-97aadf1d-efbf-4885-9c29-0d5dd74e636c tempest-ServersTestMultiNic-773389532 tempest-ServersTestMultiNic-773389532-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 21:50:14.325651 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-97aadf1d-efbf-4885-9c29-0d5dd74e636c tempest-ServersTestMultiNic-773389532 tempest-ServersTestMultiNic-773389532-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 21:50:14.326074 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-97aadf1d-efbf-4885-9c29-0d5dd74e636c tempest-ServersTestMultiNic-773389532 tempest-ServersTestMultiNic-773389532-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 21:50:14.327602 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-97aadf1d-efbf-4885-9c29-0d5dd74e636c tempest-ServersTestMultiNic-773389532 tempest-ServersTestMultiNic-773389532-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 21:50:14.332450 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-97aadf1d-efbf-4885-9c29-0d5dd74e636c tempest-ServersTestMultiNic-773389532 tempest-ServersTestMultiNic-773389532-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 21:50:14.333093 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-97aadf1d-efbf-4885-9c29-0d5dd74e636c tempest-ServersTestMultiNic-773389532 tempest-ServersTestMultiNic-773389532-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.005s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 21:50:14.350637 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-97aadf1d-efbf-4885-9c29-0d5dd74e636c tempest-ServersTestMultiNic-773389532 tempest-ServersTestMultiNic-773389532-project] Getting quotas for project 6c760d0117f747aeb13f8d85dad5c7a0. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 18 21:50:14.354214 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-97aadf1d-efbf-4885-9c29-0d5dd74e636c tempest-ServersTestMultiNic-773389532 tempest-ServersTestMultiNic-773389532-project] Getting quotas for user 7a09f989352e472697d48a8423113cdc and project 6c760d0117f747aeb13f8d85dad5c7a0. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 18 21:50:14.365813 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-97aadf1d-efbf-4885-9c29-0d5dd74e636c tempest-ServersTestMultiNic-773389532 tempest-ServersTestMultiNic-773389532-project] [instance: aab7765a-5af3-41b9-8be9-d21ceb3f7b43] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120657) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 18 21:50:14.366766 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-97aadf1d-efbf-4885-9c29-0d5dd74e636c tempest-ServersTestMultiNic-773389532 tempest-ServersTestMultiNic-773389532-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 21:50:14.367426 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-97aadf1d-efbf-4885-9c29-0d5dd74e636c tempest-ServersTestMultiNic-773389532 tempest-ServersTestMultiNic-773389532-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 21:50:14.367885 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-97aadf1d-efbf-4885-9c29-0d5dd74e636c tempest-ServersTestMultiNic-773389532 tempest-ServersTestMultiNic-773389532-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 21:50:14.431083 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-97aadf1d-efbf-4885-9c29-0d5dd74e636c tempest-ServersTestMultiNic-773389532 tempest-ServersTestMultiNic-773389532-project] [instance: aab7765a-5af3-41b9-8be9-d21ceb3f7b43] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='87b0fec9-a948-41f9-a3c0-f0a6a5a8f1e7',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120657) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 18 21:50:14.432355 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-97aadf1d-efbf-4885-9c29-0d5dd74e636c tempest-ServersTestMultiNic-773389532 tempest-ServersTestMultiNic-773389532-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 21:50:14.432799 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-97aadf1d-efbf-4885-9c29-0d5dd74e636c tempest-ServersTestMultiNic-773389532 tempest-ServersTestMultiNic-773389532-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 21:50:14.433225 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-97aadf1d-efbf-4885-9c29-0d5dd74e636c tempest-ServersTestMultiNic-773389532 tempest-ServersTestMultiNic-773389532-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 21:50:14.724749 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-97aadf1d-efbf-4885-9c29-0d5dd74e636c tempest-ServersTestMultiNic-773389532 tempest-ServersTestMultiNic-773389532-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 21:50:14.725470 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-97aadf1d-efbf-4885-9c29-0d5dd74e636c tempest-ServersTestMultiNic-773389532 tempest-ServersTestMultiNic-773389532-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 21:50:14.725873 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-97aadf1d-efbf-4885-9c29-0d5dd74e636c tempest-ServersTestMultiNic-773389532 tempest-ServersTestMultiNic-773389532-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 21:50:14.726333 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-97aadf1d-efbf-4885-9c29-0d5dd74e636c tempest-ServersTestMultiNic-773389532 tempest-ServersTestMultiNic-773389532-project] Acquiring lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 18 21:50:14.735134 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-97aadf1d-efbf-4885-9c29-0d5dd74e636c tempest-ServersTestMultiNic-773389532 tempest-ServersTestMultiNic-773389532-project] Acquired lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 18 21:50:14.735134 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c9ae31db-e787-4fbc-a257-264cae0e0276 None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 21:50:14.735134 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c9ae31db-e787-4fbc-a257-264cae0e0276 None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 21:50:14.735134 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c9ae31db-e787-4fbc-a257-264cae0e0276 None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 21:50:14.735134 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c9ae31db-e787-4fbc-a257-264cae0e0276 None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 21:50:14.735569 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c9ae31db-e787-4fbc-a257-264cae0e0276 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 21:50:14.735569 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c9ae31db-e787-4fbc-a257-264cae0e0276 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 21:50:14.738082 n-d-853704-1 nova-conductor[120657]: ERROR nova.compute.rpcapi [None req-c9ae31db-e787-4fbc-a257-264cae0e0276 None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 18 21:50:14.738785 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c9ae31db-e787-4fbc-a257-264cae0e0276 None None] Releasing lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 18 21:50:14.739767 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server [None req-c9ae31db-e787-4fbc-a257-264cae0e0276 None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 21:50:14.739767 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 21:50:14.739767 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 18 21:50:14.739767 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 18 21:50:14.739767 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 18 21:50:14.739767 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 21:50:14.739767 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 18 21:50:14.739767 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 21:50:14.739767 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 21:50:14.739767 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 18 21:50:14.739767 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 18 21:50:14.739767 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 18 21:50:14.739767 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 18 21:50:14.739767 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 18 21:50:14.739767 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 18 21:50:14.739767 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 18 21:50:14.739767 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 18 21:50:14.741111 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 18 21:50:14.741111 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 18 21:50:14.741111 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 18 21:50:14.741111 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 18 21:50:14.741111 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 18 21:50:14.741111 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 18 21:50:14.741111 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 21:50:14.741111 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 21:50:24.521436 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=13,nova_api:UPDATE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:50:24.600221 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=11,nova_api:DELETE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:50:24.736963 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:50:24.737539 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell1:SELECT=11,nova_cell1:SAVEPOINT=1,nova_cell1:INSERT=18,nova_cell1:RELEASE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:50:52.840281 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=11,nova_cell0:UPDATE=3 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:50:52.888015 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:51:12.850144 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=4,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:51:12.904220 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=4,nova_cell0:UPDATE=2 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:51:32.851535 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=4,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:51:32.902858 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=4,nova_cell0:UPDATE=2 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:52:12.860013 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:52:12.911434 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:52:42.869017 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=5,nova_cell0:UPDATE=3 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:52:52.896981 n-d-853704-1 nova-conductor[120657]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 18 21:52:52.907619 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:53:22.872048 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:53:22.911184 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=5,nova_cell0:UPDATE=3 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:54:02.873768 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=7,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:54:02.914597 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:54:12.907103 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-b4fda346-7f19-490d-8e28-376f18a99262 tempest-DeleteServersAdminTestJSON-327361073 tempest-DeleteServersAdminTestJSON-327361073-project] Took 0.85 seconds to select destinations for 1 instance(s). {{(pid=120658) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 18 21:54:12.933860 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-b4fda346-7f19-490d-8e28-376f18a99262 tempest-DeleteServersAdminTestJSON-327361073 tempest-DeleteServersAdminTestJSON-327361073-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 21:54:12.934301 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-b4fda346-7f19-490d-8e28-376f18a99262 tempest-DeleteServersAdminTestJSON-327361073 tempest-DeleteServersAdminTestJSON-327361073-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 21:54:12.934694 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-b4fda346-7f19-490d-8e28-376f18a99262 tempest-DeleteServersAdminTestJSON-327361073 tempest-DeleteServersAdminTestJSON-327361073-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 21:54:13.176375 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-b4fda346-7f19-490d-8e28-376f18a99262 tempest-DeleteServersAdminTestJSON-327361073 tempest-DeleteServersAdminTestJSON-327361073-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 21:54:13.177266 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-b4fda346-7f19-490d-8e28-376f18a99262 tempest-DeleteServersAdminTestJSON-327361073 tempest-DeleteServersAdminTestJSON-327361073-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 21:54:13.177681 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-b4fda346-7f19-490d-8e28-376f18a99262 tempest-DeleteServersAdminTestJSON-327361073 tempest-DeleteServersAdminTestJSON-327361073-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 21:54:13.178401 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-b4fda346-7f19-490d-8e28-376f18a99262 tempest-DeleteServersAdminTestJSON-327361073 tempest-DeleteServersAdminTestJSON-327361073-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 21:54:13.178833 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-b4fda346-7f19-490d-8e28-376f18a99262 tempest-DeleteServersAdminTestJSON-327361073 tempest-DeleteServersAdminTestJSON-327361073-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 21:54:13.179210 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-b4fda346-7f19-490d-8e28-376f18a99262 tempest-DeleteServersAdminTestJSON-327361073 tempest-DeleteServersAdminTestJSON-327361073-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 21:54:13.196363 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-b4fda346-7f19-490d-8e28-376f18a99262 tempest-DeleteServersAdminTestJSON-327361073 tempest-DeleteServersAdminTestJSON-327361073-project] Getting quotas for project 4538a3db5b634db3b684656d07abe905. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 18 21:54:13.210996 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-b4fda346-7f19-490d-8e28-376f18a99262 tempest-DeleteServersAdminTestJSON-327361073 tempest-DeleteServersAdminTestJSON-327361073-project] Getting quotas for user 4e77bbbfeef840929431a8600eabbe5e and project 4538a3db5b634db3b684656d07abe905. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 18 21:54:13.211241 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-b4fda346-7f19-490d-8e28-376f18a99262 tempest-DeleteServersAdminTestJSON-327361073 tempest-DeleteServersAdminTestJSON-327361073-project] [instance: 25dff9d3-d1ed-45b3-a947-766385bdfc62] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120658) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 18 21:54:13.212065 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-b4fda346-7f19-490d-8e28-376f18a99262 tempest-DeleteServersAdminTestJSON-327361073 tempest-DeleteServersAdminTestJSON-327361073-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 21:54:13.212504 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-b4fda346-7f19-490d-8e28-376f18a99262 tempest-DeleteServersAdminTestJSON-327361073 tempest-DeleteServersAdminTestJSON-327361073-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 21:54:13.212898 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-b4fda346-7f19-490d-8e28-376f18a99262 tempest-DeleteServersAdminTestJSON-327361073 tempest-DeleteServersAdminTestJSON-327361073-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 21:54:13.310420 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-b4fda346-7f19-490d-8e28-376f18a99262 tempest-DeleteServersAdminTestJSON-327361073 tempest-DeleteServersAdminTestJSON-327361073-project] [instance: 25dff9d3-d1ed-45b3-a947-766385bdfc62] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='87b0fec9-a948-41f9-a3c0-f0a6a5a8f1e7',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120658) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 18 21:54:13.312564 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-b4fda346-7f19-490d-8e28-376f18a99262 tempest-DeleteServersAdminTestJSON-327361073 tempest-DeleteServersAdminTestJSON-327361073-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 21:54:13.313330 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-b4fda346-7f19-490d-8e28-376f18a99262 tempest-DeleteServersAdminTestJSON-327361073 tempest-DeleteServersAdminTestJSON-327361073-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 21:54:13.313741 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-b4fda346-7f19-490d-8e28-376f18a99262 tempest-DeleteServersAdminTestJSON-327361073 tempest-DeleteServersAdminTestJSON-327361073-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 21:54:13.616538 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-b4fda346-7f19-490d-8e28-376f18a99262 tempest-DeleteServersAdminTestJSON-327361073 tempest-DeleteServersAdminTestJSON-327361073-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 21:54:13.617431 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-b4fda346-7f19-490d-8e28-376f18a99262 tempest-DeleteServersAdminTestJSON-327361073 tempest-DeleteServersAdminTestJSON-327361073-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 21:54:13.618578 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-b4fda346-7f19-490d-8e28-376f18a99262 tempest-DeleteServersAdminTestJSON-327361073 tempest-DeleteServersAdminTestJSON-327361073-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 21:54:13.619261 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-b4fda346-7f19-490d-8e28-376f18a99262 tempest-DeleteServersAdminTestJSON-327361073 tempest-DeleteServersAdminTestJSON-327361073-project] Acquiring lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 18 21:54:13.619931 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-b4fda346-7f19-490d-8e28-376f18a99262 tempest-DeleteServersAdminTestJSON-327361073 tempest-DeleteServersAdminTestJSON-327361073-project] Acquired lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 18 21:54:13.621039 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-6d71bb5b-39de-496f-aac5-0b15637cde63 None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 21:54:13.623737 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-6d71bb5b-39de-496f-aac5-0b15637cde63 None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 21:54:13.624183 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-6d71bb5b-39de-496f-aac5-0b15637cde63 None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.002s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 21:54:13.625833 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-6d71bb5b-39de-496f-aac5-0b15637cde63 None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 21:54:13.626774 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-6d71bb5b-39de-496f-aac5-0b15637cde63 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 21:54:13.627258 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-6d71bb5b-39de-496f-aac5-0b15637cde63 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 21:54:13.637893 n-d-853704-1 nova-conductor[120658]: ERROR nova.compute.rpcapi [None req-6d71bb5b-39de-496f-aac5-0b15637cde63 None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 18 21:54:13.638283 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-6d71bb5b-39de-496f-aac5-0b15637cde63 None None] Releasing lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 18 21:54:13.639381 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server [None req-6d71bb5b-39de-496f-aac5-0b15637cde63 None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 21:54:13.639381 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 21:54:13.639381 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 18 21:54:13.639381 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 18 21:54:13.639381 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 18 21:54:13.639381 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 21:54:13.639381 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 18 21:54:13.639381 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 21:54:13.639381 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 21:54:13.639381 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 18 21:54:13.639381 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 18 21:54:13.639381 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 18 21:54:13.639381 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 18 21:54:13.639381 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 18 21:54:13.639381 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 18 21:54:13.639381 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 18 21:54:13.639381 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 18 21:54:13.640657 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 18 21:54:13.640657 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 18 21:54:13.640657 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 18 21:54:13.640657 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 18 21:54:13.640657 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 18 21:54:13.640657 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 18 21:54:13.640657 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 21:54:13.640657 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 21:54:23.427532 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=13 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:54:23.520841 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=11,nova_api:UPDATE=1,nova_api:DELETE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:54:23.635104 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:54:23.635995 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell1:SELECT=11,nova_cell1:SAVEPOINT=1,nova_cell1:INSERT=18,nova_cell1:RELEASE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:54:23.852518 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-c8bb88a5-32f9-4363-930b-47bda989fff5 tempest-LiveMigrationTest-1883958412 tempest-LiveMigrationTest-1883958412-project] Took 0.45 seconds to select destinations for 1 instance(s). {{(pid=120657) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 18 21:54:23.873129 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c8bb88a5-32f9-4363-930b-47bda989fff5 tempest-LiveMigrationTest-1883958412 tempest-LiveMigrationTest-1883958412-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 21:54:23.873608 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c8bb88a5-32f9-4363-930b-47bda989fff5 tempest-LiveMigrationTest-1883958412 tempest-LiveMigrationTest-1883958412-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 21:54:23.874005 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c8bb88a5-32f9-4363-930b-47bda989fff5 tempest-LiveMigrationTest-1883958412 tempest-LiveMigrationTest-1883958412-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 21:54:24.002803 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c8bb88a5-32f9-4363-930b-47bda989fff5 tempest-LiveMigrationTest-1883958412 tempest-LiveMigrationTest-1883958412-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 21:54:24.003273 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c8bb88a5-32f9-4363-930b-47bda989fff5 tempest-LiveMigrationTest-1883958412 tempest-LiveMigrationTest-1883958412-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 21:54:24.003685 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c8bb88a5-32f9-4363-930b-47bda989fff5 tempest-LiveMigrationTest-1883958412 tempest-LiveMigrationTest-1883958412-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 21:54:24.004347 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c8bb88a5-32f9-4363-930b-47bda989fff5 tempest-LiveMigrationTest-1883958412 tempest-LiveMigrationTest-1883958412-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 21:54:24.004757 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c8bb88a5-32f9-4363-930b-47bda989fff5 tempest-LiveMigrationTest-1883958412 tempest-LiveMigrationTest-1883958412-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 21:54:24.005139 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c8bb88a5-32f9-4363-930b-47bda989fff5 tempest-LiveMigrationTest-1883958412 tempest-LiveMigrationTest-1883958412-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 21:54:24.020572 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-c8bb88a5-32f9-4363-930b-47bda989fff5 tempest-LiveMigrationTest-1883958412 tempest-LiveMigrationTest-1883958412-project] Getting quotas for project 29de488917ec43a09800ec6cb2187b56. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 18 21:54:24.024681 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-c8bb88a5-32f9-4363-930b-47bda989fff5 tempest-LiveMigrationTest-1883958412 tempest-LiveMigrationTest-1883958412-project] Getting quotas for user c137b718cd2445909cacb17658bff8f6 and project 29de488917ec43a09800ec6cb2187b56. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 18 21:54:24.035849 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-c8bb88a5-32f9-4363-930b-47bda989fff5 tempest-LiveMigrationTest-1883958412 tempest-LiveMigrationTest-1883958412-project] [instance: 805e50fe-dca3-4e5b-a798-5b0450dcfcf2] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120657) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 18 21:54:24.036590 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c8bb88a5-32f9-4363-930b-47bda989fff5 tempest-LiveMigrationTest-1883958412 tempest-LiveMigrationTest-1883958412-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 21:54:24.037018 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c8bb88a5-32f9-4363-930b-47bda989fff5 tempest-LiveMigrationTest-1883958412 tempest-LiveMigrationTest-1883958412-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 21:54:24.037406 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c8bb88a5-32f9-4363-930b-47bda989fff5 tempest-LiveMigrationTest-1883958412 tempest-LiveMigrationTest-1883958412-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 21:54:24.117988 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-c8bb88a5-32f9-4363-930b-47bda989fff5 tempest-LiveMigrationTest-1883958412 tempest-LiveMigrationTest-1883958412-project] [instance: 805e50fe-dca3-4e5b-a798-5b0450dcfcf2] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='87b0fec9-a948-41f9-a3c0-f0a6a5a8f1e7',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120657) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 18 21:54:24.119049 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c8bb88a5-32f9-4363-930b-47bda989fff5 tempest-LiveMigrationTest-1883958412 tempest-LiveMigrationTest-1883958412-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 21:54:24.119512 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c8bb88a5-32f9-4363-930b-47bda989fff5 tempest-LiveMigrationTest-1883958412 tempest-LiveMigrationTest-1883958412-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 21:54:24.119951 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c8bb88a5-32f9-4363-930b-47bda989fff5 tempest-LiveMigrationTest-1883958412 tempest-LiveMigrationTest-1883958412-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 21:54:24.438232 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c8bb88a5-32f9-4363-930b-47bda989fff5 tempest-LiveMigrationTest-1883958412 tempest-LiveMigrationTest-1883958412-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 21:54:24.438820 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c8bb88a5-32f9-4363-930b-47bda989fff5 tempest-LiveMigrationTest-1883958412 tempest-LiveMigrationTest-1883958412-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 21:54:24.439216 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c8bb88a5-32f9-4363-930b-47bda989fff5 tempest-LiveMigrationTest-1883958412 tempest-LiveMigrationTest-1883958412-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 21:54:24.439687 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c8bb88a5-32f9-4363-930b-47bda989fff5 tempest-LiveMigrationTest-1883958412 tempest-LiveMigrationTest-1883958412-project] Acquiring lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 18 21:54:24.446457 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c8bb88a5-32f9-4363-930b-47bda989fff5 tempest-LiveMigrationTest-1883958412 tempest-LiveMigrationTest-1883958412-project] Acquired lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 18 21:54:24.446457 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-418fb841-50b1-4cc7-b11c-89a871eedc2e None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 21:54:24.446457 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-418fb841-50b1-4cc7-b11c-89a871eedc2e None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 21:54:24.446457 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-418fb841-50b1-4cc7-b11c-89a871eedc2e None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 21:54:24.446457 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-418fb841-50b1-4cc7-b11c-89a871eedc2e None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 21:54:24.446878 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-418fb841-50b1-4cc7-b11c-89a871eedc2e None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 21:54:24.446878 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-418fb841-50b1-4cc7-b11c-89a871eedc2e None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 21:54:24.452005 n-d-853704-1 nova-conductor[120657]: ERROR nova.compute.rpcapi [None req-418fb841-50b1-4cc7-b11c-89a871eedc2e None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 18 21:54:24.452394 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-418fb841-50b1-4cc7-b11c-89a871eedc2e None None] Releasing lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 18 21:54:24.453847 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server [None req-418fb841-50b1-4cc7-b11c-89a871eedc2e None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 21:54:24.453847 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 21:54:24.453847 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 18 21:54:24.453847 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 18 21:54:24.453847 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 18 21:54:24.453847 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 21:54:24.453847 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 18 21:54:24.453847 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 21:54:24.453847 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 21:54:24.453847 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 18 21:54:24.453847 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 18 21:54:24.453847 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 18 21:54:24.453847 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 18 21:54:24.453847 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 18 21:54:24.453847 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 18 21:54:24.453847 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 18 21:54:24.453847 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 18 21:54:24.455075 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 18 21:54:24.455075 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 18 21:54:24.455075 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 18 21:54:24.455075 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 18 21:54:24.455075 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 18 21:54:24.455075 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 18 21:54:24.455075 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 21:54:24.455075 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 21:54:32.922177 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=12,nova_cell0:UPDATE=3 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:54:34.198218 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=10 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:54:34.310688 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=14,nova_api:UPDATE=1,nova_api:DELETE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:54:34.450843 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:54:34.451643 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell1:SELECT=11,nova_cell1:INSERT=17 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:54:38.233943 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-16ea6762-2b8d-41de-bb61-c6ea57ab3053 tempest-FloatingIPsAssociationTestJSON-1321153739 tempest-FloatingIPsAssociationTestJSON-1321153739-project] Took 0.64 seconds to select destinations for 1 instance(s). {{(pid=120658) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 18 21:54:38.257407 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-16ea6762-2b8d-41de-bb61-c6ea57ab3053 tempest-FloatingIPsAssociationTestJSON-1321153739 tempest-FloatingIPsAssociationTestJSON-1321153739-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 21:54:38.260668 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-16ea6762-2b8d-41de-bb61-c6ea57ab3053 tempest-FloatingIPsAssociationTestJSON-1321153739 tempest-FloatingIPsAssociationTestJSON-1321153739-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 21:54:38.260668 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-16ea6762-2b8d-41de-bb61-c6ea57ab3053 tempest-FloatingIPsAssociationTestJSON-1321153739 tempest-FloatingIPsAssociationTestJSON-1321153739-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 21:54:38.475791 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-16ea6762-2b8d-41de-bb61-c6ea57ab3053 tempest-FloatingIPsAssociationTestJSON-1321153739 tempest-FloatingIPsAssociationTestJSON-1321153739-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 21:54:38.476268 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-16ea6762-2b8d-41de-bb61-c6ea57ab3053 tempest-FloatingIPsAssociationTestJSON-1321153739 tempest-FloatingIPsAssociationTestJSON-1321153739-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 21:54:38.476812 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-16ea6762-2b8d-41de-bb61-c6ea57ab3053 tempest-FloatingIPsAssociationTestJSON-1321153739 tempest-FloatingIPsAssociationTestJSON-1321153739-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 21:54:38.477572 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-16ea6762-2b8d-41de-bb61-c6ea57ab3053 tempest-FloatingIPsAssociationTestJSON-1321153739 tempest-FloatingIPsAssociationTestJSON-1321153739-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 21:54:38.478052 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-16ea6762-2b8d-41de-bb61-c6ea57ab3053 tempest-FloatingIPsAssociationTestJSON-1321153739 tempest-FloatingIPsAssociationTestJSON-1321153739-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 21:54:38.478500 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-16ea6762-2b8d-41de-bb61-c6ea57ab3053 tempest-FloatingIPsAssociationTestJSON-1321153739 tempest-FloatingIPsAssociationTestJSON-1321153739-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 21:54:38.496469 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-16ea6762-2b8d-41de-bb61-c6ea57ab3053 tempest-FloatingIPsAssociationTestJSON-1321153739 tempest-FloatingIPsAssociationTestJSON-1321153739-project] Getting quotas for project 3b0efe0e0d2544ada74773b36eb5b414. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 18 21:54:38.501296 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-16ea6762-2b8d-41de-bb61-c6ea57ab3053 tempest-FloatingIPsAssociationTestJSON-1321153739 tempest-FloatingIPsAssociationTestJSON-1321153739-project] Getting quotas for user aa733f126d75484aaa0a8ef724c464e1 and project 3b0efe0e0d2544ada74773b36eb5b414. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 18 21:54:38.514280 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-16ea6762-2b8d-41de-bb61-c6ea57ab3053 tempest-FloatingIPsAssociationTestJSON-1321153739 tempest-FloatingIPsAssociationTestJSON-1321153739-project] [instance: b2b42107-5f74-4977-83c3-c655d1eceab9] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120658) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 18 21:54:38.515183 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-16ea6762-2b8d-41de-bb61-c6ea57ab3053 tempest-FloatingIPsAssociationTestJSON-1321153739 tempest-FloatingIPsAssociationTestJSON-1321153739-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 21:54:38.515837 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-16ea6762-2b8d-41de-bb61-c6ea57ab3053 tempest-FloatingIPsAssociationTestJSON-1321153739 tempest-FloatingIPsAssociationTestJSON-1321153739-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 21:54:38.516257 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-16ea6762-2b8d-41de-bb61-c6ea57ab3053 tempest-FloatingIPsAssociationTestJSON-1321153739 tempest-FloatingIPsAssociationTestJSON-1321153739-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 21:54:38.678330 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-16ea6762-2b8d-41de-bb61-c6ea57ab3053 tempest-FloatingIPsAssociationTestJSON-1321153739 tempest-FloatingIPsAssociationTestJSON-1321153739-project] [instance: b2b42107-5f74-4977-83c3-c655d1eceab9] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='87b0fec9-a948-41f9-a3c0-f0a6a5a8f1e7',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120658) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 18 21:54:38.680458 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-16ea6762-2b8d-41de-bb61-c6ea57ab3053 tempest-FloatingIPsAssociationTestJSON-1321153739 tempest-FloatingIPsAssociationTestJSON-1321153739-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 21:54:38.681229 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-16ea6762-2b8d-41de-bb61-c6ea57ab3053 tempest-FloatingIPsAssociationTestJSON-1321153739 tempest-FloatingIPsAssociationTestJSON-1321153739-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 21:54:38.681626 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-16ea6762-2b8d-41de-bb61-c6ea57ab3053 tempest-FloatingIPsAssociationTestJSON-1321153739 tempest-FloatingIPsAssociationTestJSON-1321153739-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 21:54:39.145656 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-16ea6762-2b8d-41de-bb61-c6ea57ab3053 tempest-FloatingIPsAssociationTestJSON-1321153739 tempest-FloatingIPsAssociationTestJSON-1321153739-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 21:54:39.146099 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-16ea6762-2b8d-41de-bb61-c6ea57ab3053 tempest-FloatingIPsAssociationTestJSON-1321153739 tempest-FloatingIPsAssociationTestJSON-1321153739-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 21:54:39.146492 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-16ea6762-2b8d-41de-bb61-c6ea57ab3053 tempest-FloatingIPsAssociationTestJSON-1321153739 tempest-FloatingIPsAssociationTestJSON-1321153739-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 21:54:39.146992 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-16ea6762-2b8d-41de-bb61-c6ea57ab3053 tempest-FloatingIPsAssociationTestJSON-1321153739 tempest-FloatingIPsAssociationTestJSON-1321153739-project] Acquiring lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 18 21:54:39.147499 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-16ea6762-2b8d-41de-bb61-c6ea57ab3053 tempest-FloatingIPsAssociationTestJSON-1321153739 tempest-FloatingIPsAssociationTestJSON-1321153739-project] Acquired lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 18 21:54:39.148272 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-6af826ca-a850-4783-ba94-5c5dd37c8ab8 None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 21:54:39.153781 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-6af826ca-a850-4783-ba94-5c5dd37c8ab8 None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 21:54:39.153781 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-6af826ca-a850-4783-ba94-5c5dd37c8ab8 None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 21:54:39.153781 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-6af826ca-a850-4783-ba94-5c5dd37c8ab8 None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 21:54:39.153781 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-6af826ca-a850-4783-ba94-5c5dd37c8ab8 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 21:54:39.153781 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-6af826ca-a850-4783-ba94-5c5dd37c8ab8 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 21:54:39.161611 n-d-853704-1 nova-conductor[120658]: ERROR nova.compute.rpcapi [None req-6af826ca-a850-4783-ba94-5c5dd37c8ab8 None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 18 21:54:39.162002 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-6af826ca-a850-4783-ba94-5c5dd37c8ab8 None None] Releasing lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 18 21:54:39.162909 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server [None req-6af826ca-a850-4783-ba94-5c5dd37c8ab8 None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 21:54:39.162909 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 21:54:39.162909 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 18 21:54:39.162909 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 18 21:54:39.162909 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 18 21:54:39.162909 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 21:54:39.162909 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 18 21:54:39.162909 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 21:54:39.162909 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 21:54:39.162909 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 18 21:54:39.162909 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 18 21:54:39.162909 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 18 21:54:39.162909 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 18 21:54:39.162909 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 18 21:54:39.162909 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 18 21:54:39.162909 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 18 21:54:39.162909 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 18 21:54:39.164256 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 18 21:54:39.164256 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 18 21:54:39.164256 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 18 21:54:39.164256 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 18 21:54:39.164256 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 18 21:54:39.164256 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 18 21:54:39.164256 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 21:54:39.164256 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 21:54:42.877038 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=15,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:54:48.863549 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=13 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:54:49.001827 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=11,nova_api:UPDATE=1,nova_api:DELETE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:54:49.161201 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:54:49.161923 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell1:SELECT=11,nova_cell1:SAVEPOINT=1,nova_cell1:INSERT=18,nova_cell1:RELEASE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:54:53.899233 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-a3f97d59-5de5-48c4-87e8-b849db12877f tempest-LiveAutoBlockMigrationV225Test-1869140755 tempest-LiveAutoBlockMigrationV225Test-1869140755-project] Took 0.61 seconds to select destinations for 1 instance(s). {{(pid=120657) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 18 21:54:53.928959 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-a3f97d59-5de5-48c4-87e8-b849db12877f tempest-LiveAutoBlockMigrationV225Test-1869140755 tempest-LiveAutoBlockMigrationV225Test-1869140755-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 21:54:53.929524 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-a3f97d59-5de5-48c4-87e8-b849db12877f tempest-LiveAutoBlockMigrationV225Test-1869140755 tempest-LiveAutoBlockMigrationV225Test-1869140755-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 21:54:53.930352 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-a3f97d59-5de5-48c4-87e8-b849db12877f tempest-LiveAutoBlockMigrationV225Test-1869140755 tempest-LiveAutoBlockMigrationV225Test-1869140755-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 21:54:54.102559 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-a3f97d59-5de5-48c4-87e8-b849db12877f tempest-LiveAutoBlockMigrationV225Test-1869140755 tempest-LiveAutoBlockMigrationV225Test-1869140755-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 21:54:54.102994 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-a3f97d59-5de5-48c4-87e8-b849db12877f tempest-LiveAutoBlockMigrationV225Test-1869140755 tempest-LiveAutoBlockMigrationV225Test-1869140755-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 21:54:54.103385 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-a3f97d59-5de5-48c4-87e8-b849db12877f tempest-LiveAutoBlockMigrationV225Test-1869140755 tempest-LiveAutoBlockMigrationV225Test-1869140755-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 21:54:54.104067 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-a3f97d59-5de5-48c4-87e8-b849db12877f tempest-LiveAutoBlockMigrationV225Test-1869140755 tempest-LiveAutoBlockMigrationV225Test-1869140755-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 21:54:54.104471 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-a3f97d59-5de5-48c4-87e8-b849db12877f tempest-LiveAutoBlockMigrationV225Test-1869140755 tempest-LiveAutoBlockMigrationV225Test-1869140755-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 21:54:54.104859 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-a3f97d59-5de5-48c4-87e8-b849db12877f tempest-LiveAutoBlockMigrationV225Test-1869140755 tempest-LiveAutoBlockMigrationV225Test-1869140755-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 21:54:54.123290 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-a3f97d59-5de5-48c4-87e8-b849db12877f tempest-LiveAutoBlockMigrationV225Test-1869140755 tempest-LiveAutoBlockMigrationV225Test-1869140755-project] Getting quotas for project 4c471d428ca241faa5d631d226526b3b. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 18 21:54:54.127947 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-a3f97d59-5de5-48c4-87e8-b849db12877f tempest-LiveAutoBlockMigrationV225Test-1869140755 tempest-LiveAutoBlockMigrationV225Test-1869140755-project] Getting quotas for user faee7152cc454a189fcbdd8f2d4b7b29 and project 4c471d428ca241faa5d631d226526b3b. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 18 21:54:54.139951 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-a3f97d59-5de5-48c4-87e8-b849db12877f tempest-LiveAutoBlockMigrationV225Test-1869140755 tempest-LiveAutoBlockMigrationV225Test-1869140755-project] [instance: d1dedc17-835f-484c-9d84-4f4f29cb6410] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120657) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 18 21:54:54.140770 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-a3f97d59-5de5-48c4-87e8-b849db12877f tempest-LiveAutoBlockMigrationV225Test-1869140755 tempest-LiveAutoBlockMigrationV225Test-1869140755-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 21:54:54.141208 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-a3f97d59-5de5-48c4-87e8-b849db12877f tempest-LiveAutoBlockMigrationV225Test-1869140755 tempest-LiveAutoBlockMigrationV225Test-1869140755-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 21:54:54.141604 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-a3f97d59-5de5-48c4-87e8-b849db12877f tempest-LiveAutoBlockMigrationV225Test-1869140755 tempest-LiveAutoBlockMigrationV225Test-1869140755-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 21:54:54.233723 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-a3f97d59-5de5-48c4-87e8-b849db12877f tempest-LiveAutoBlockMigrationV225Test-1869140755 tempest-LiveAutoBlockMigrationV225Test-1869140755-project] [instance: d1dedc17-835f-484c-9d84-4f4f29cb6410] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='87b0fec9-a948-41f9-a3c0-f0a6a5a8f1e7',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120657) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 18 21:54:54.236583 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-a3f97d59-5de5-48c4-87e8-b849db12877f tempest-LiveAutoBlockMigrationV225Test-1869140755 tempest-LiveAutoBlockMigrationV225Test-1869140755-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 21:54:54.238297 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-a3f97d59-5de5-48c4-87e8-b849db12877f tempest-LiveAutoBlockMigrationV225Test-1869140755 tempest-LiveAutoBlockMigrationV225Test-1869140755-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.002s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 21:54:54.239825 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-a3f97d59-5de5-48c4-87e8-b849db12877f tempest-LiveAutoBlockMigrationV225Test-1869140755 tempest-LiveAutoBlockMigrationV225Test-1869140755-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 21:54:54.631640 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-a3f97d59-5de5-48c4-87e8-b849db12877f tempest-LiveAutoBlockMigrationV225Test-1869140755 tempest-LiveAutoBlockMigrationV225Test-1869140755-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 21:54:54.632560 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-a3f97d59-5de5-48c4-87e8-b849db12877f tempest-LiveAutoBlockMigrationV225Test-1869140755 tempest-LiveAutoBlockMigrationV225Test-1869140755-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 21:54:54.633254 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-a3f97d59-5de5-48c4-87e8-b849db12877f tempest-LiveAutoBlockMigrationV225Test-1869140755 tempest-LiveAutoBlockMigrationV225Test-1869140755-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 21:54:54.634126 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-a3f97d59-5de5-48c4-87e8-b849db12877f tempest-LiveAutoBlockMigrationV225Test-1869140755 tempest-LiveAutoBlockMigrationV225Test-1869140755-project] Acquiring lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 18 21:54:54.634793 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-a3f97d59-5de5-48c4-87e8-b849db12877f tempest-LiveAutoBlockMigrationV225Test-1869140755 tempest-LiveAutoBlockMigrationV225Test-1869140755-project] Acquired lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 18 21:54:54.636132 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c2a2eb04-a5db-4107-a8ea-650e6ff05d00 None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 21:54:54.638124 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c2a2eb04-a5db-4107-a8ea-650e6ff05d00 None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.002s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 21:54:54.644764 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c2a2eb04-a5db-4107-a8ea-650e6ff05d00 None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 21:54:54.644764 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c2a2eb04-a5db-4107-a8ea-650e6ff05d00 None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 21:54:54.644764 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c2a2eb04-a5db-4107-a8ea-650e6ff05d00 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 21:54:54.644764 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c2a2eb04-a5db-4107-a8ea-650e6ff05d00 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 21:54:54.661906 n-d-853704-1 nova-conductor[120657]: ERROR nova.compute.rpcapi [None req-c2a2eb04-a5db-4107-a8ea-650e6ff05d00 None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 18 21:54:54.665816 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c2a2eb04-a5db-4107-a8ea-650e6ff05d00 None None] Releasing lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 18 21:54:54.666870 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server [None req-c2a2eb04-a5db-4107-a8ea-650e6ff05d00 None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 21:54:54.666870 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 21:54:54.666870 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 18 21:54:54.666870 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 18 21:54:54.666870 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 18 21:54:54.666870 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 21:54:54.666870 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 18 21:54:54.666870 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 21:54:54.666870 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 21:54:54.666870 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 18 21:54:54.666870 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 18 21:54:54.666870 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 18 21:54:54.666870 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 18 21:54:54.666870 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 18 21:54:54.666870 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 18 21:54:54.666870 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 18 21:54:54.666870 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 18 21:54:54.669234 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 18 21:54:54.669234 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 18 21:54:54.669234 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 18 21:54:54.669234 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 18 21:54:54.669234 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 18 21:54:54.669234 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 18 21:54:54.669234 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 21:54:54.669234 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 21:55:03.296757 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-60b2b08a-9087-4246-b246-bad730016856 tempest-ServerDiagnosticsNegativeTest-1185055790 tempest-ServerDiagnosticsNegativeTest-1185055790-project] Took 0.82 seconds to select destinations for 1 instance(s). {{(pid=120658) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 18 21:55:03.316796 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-60b2b08a-9087-4246-b246-bad730016856 tempest-ServerDiagnosticsNegativeTest-1185055790 tempest-ServerDiagnosticsNegativeTest-1185055790-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 21:55:03.317238 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-60b2b08a-9087-4246-b246-bad730016856 tempest-ServerDiagnosticsNegativeTest-1185055790 tempest-ServerDiagnosticsNegativeTest-1185055790-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 21:55:03.322764 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-60b2b08a-9087-4246-b246-bad730016856 tempest-ServerDiagnosticsNegativeTest-1185055790 tempest-ServerDiagnosticsNegativeTest-1185055790-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 21:55:03.600479 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=15,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:55:03.604219 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-60b2b08a-9087-4246-b246-bad730016856 tempest-ServerDiagnosticsNegativeTest-1185055790 tempest-ServerDiagnosticsNegativeTest-1185055790-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 21:55:03.605217 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-60b2b08a-9087-4246-b246-bad730016856 tempest-ServerDiagnosticsNegativeTest-1185055790 tempest-ServerDiagnosticsNegativeTest-1185055790-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 21:55:03.606237 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-60b2b08a-9087-4246-b246-bad730016856 tempest-ServerDiagnosticsNegativeTest-1185055790 tempest-ServerDiagnosticsNegativeTest-1185055790-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 21:55:03.607019 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-60b2b08a-9087-4246-b246-bad730016856 tempest-ServerDiagnosticsNegativeTest-1185055790 tempest-ServerDiagnosticsNegativeTest-1185055790-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 21:55:03.607625 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-60b2b08a-9087-4246-b246-bad730016856 tempest-ServerDiagnosticsNegativeTest-1185055790 tempest-ServerDiagnosticsNegativeTest-1185055790-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 21:55:03.608096 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-60b2b08a-9087-4246-b246-bad730016856 tempest-ServerDiagnosticsNegativeTest-1185055790 tempest-ServerDiagnosticsNegativeTest-1185055790-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 21:55:03.635330 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-60b2b08a-9087-4246-b246-bad730016856 tempest-ServerDiagnosticsNegativeTest-1185055790 tempest-ServerDiagnosticsNegativeTest-1185055790-project] Getting quotas for project cbae817da5a44906be459a960be6edea. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 18 21:55:03.640189 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-60b2b08a-9087-4246-b246-bad730016856 tempest-ServerDiagnosticsNegativeTest-1185055790 tempest-ServerDiagnosticsNegativeTest-1185055790-project] Getting quotas for user c5d469421ac445a88c8e7a58d4627b63 and project cbae817da5a44906be459a960be6edea. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 18 21:55:03.653585 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-60b2b08a-9087-4246-b246-bad730016856 tempest-ServerDiagnosticsNegativeTest-1185055790 tempest-ServerDiagnosticsNegativeTest-1185055790-project] [instance: 2078f05d-cb6c-4a1f-b932-1a1f0955a439] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120658) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 18 21:55:03.654773 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-60b2b08a-9087-4246-b246-bad730016856 tempest-ServerDiagnosticsNegativeTest-1185055790 tempest-ServerDiagnosticsNegativeTest-1185055790-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 21:55:03.655425 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-60b2b08a-9087-4246-b246-bad730016856 tempest-ServerDiagnosticsNegativeTest-1185055790 tempest-ServerDiagnosticsNegativeTest-1185055790-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 21:55:03.656097 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-60b2b08a-9087-4246-b246-bad730016856 tempest-ServerDiagnosticsNegativeTest-1185055790 tempest-ServerDiagnosticsNegativeTest-1185055790-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 21:55:03.780994 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-60b2b08a-9087-4246-b246-bad730016856 tempest-ServerDiagnosticsNegativeTest-1185055790 tempest-ServerDiagnosticsNegativeTest-1185055790-project] [instance: 2078f05d-cb6c-4a1f-b932-1a1f0955a439] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='87b0fec9-a948-41f9-a3c0-f0a6a5a8f1e7',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120658) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 18 21:55:03.790919 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-60b2b08a-9087-4246-b246-bad730016856 tempest-ServerDiagnosticsNegativeTest-1185055790 tempest-ServerDiagnosticsNegativeTest-1185055790-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 21:55:03.790919 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-60b2b08a-9087-4246-b246-bad730016856 tempest-ServerDiagnosticsNegativeTest-1185055790 tempest-ServerDiagnosticsNegativeTest-1185055790-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 21:55:03.790919 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-60b2b08a-9087-4246-b246-bad730016856 tempest-ServerDiagnosticsNegativeTest-1185055790 tempest-ServerDiagnosticsNegativeTest-1185055790-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 21:55:04.045359 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-60b2b08a-9087-4246-b246-bad730016856 tempest-ServerDiagnosticsNegativeTest-1185055790 tempest-ServerDiagnosticsNegativeTest-1185055790-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 21:55:04.045802 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-60b2b08a-9087-4246-b246-bad730016856 tempest-ServerDiagnosticsNegativeTest-1185055790 tempest-ServerDiagnosticsNegativeTest-1185055790-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 21:55:04.046237 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-60b2b08a-9087-4246-b246-bad730016856 tempest-ServerDiagnosticsNegativeTest-1185055790 tempest-ServerDiagnosticsNegativeTest-1185055790-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 21:55:04.046691 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-60b2b08a-9087-4246-b246-bad730016856 tempest-ServerDiagnosticsNegativeTest-1185055790 tempest-ServerDiagnosticsNegativeTest-1185055790-project] Acquiring lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 18 21:55:04.047044 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-60b2b08a-9087-4246-b246-bad730016856 tempest-ServerDiagnosticsNegativeTest-1185055790 tempest-ServerDiagnosticsNegativeTest-1185055790-project] Acquired lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 18 21:55:04.047801 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-0a27edd6-4255-470c-b7ef-5d283bd9f538 None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 21:55:04.048219 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-0a27edd6-4255-470c-b7ef-5d283bd9f538 None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 21:55:04.048607 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-0a27edd6-4255-470c-b7ef-5d283bd9f538 None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 21:55:04.049257 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-0a27edd6-4255-470c-b7ef-5d283bd9f538 None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 21:55:04.049724 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-0a27edd6-4255-470c-b7ef-5d283bd9f538 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 21:55:04.050114 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-0a27edd6-4255-470c-b7ef-5d283bd9f538 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 21:55:04.058087 n-d-853704-1 nova-conductor[120658]: ERROR nova.compute.rpcapi [None req-0a27edd6-4255-470c-b7ef-5d283bd9f538 None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 18 21:55:04.058479 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-0a27edd6-4255-470c-b7ef-5d283bd9f538 None None] Releasing lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 18 21:55:04.059903 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server [None req-0a27edd6-4255-470c-b7ef-5d283bd9f538 None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 21:55:04.059903 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 21:55:04.059903 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 18 21:55:04.059903 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 18 21:55:04.059903 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 18 21:55:04.059903 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 21:55:04.059903 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 18 21:55:04.059903 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 21:55:04.059903 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 21:55:04.059903 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 18 21:55:04.059903 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 18 21:55:04.059903 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 18 21:55:04.059903 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 18 21:55:04.059903 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 18 21:55:04.059903 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 18 21:55:04.059903 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 18 21:55:04.059903 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 18 21:55:04.061069 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 18 21:55:04.061069 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 18 21:55:04.061069 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 18 21:55:04.061069 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 18 21:55:04.061069 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 18 21:55:04.061069 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 18 21:55:04.061069 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 21:55:04.061069 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 21:55:04.362740 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=13 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:55:04.659118 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:55:04.660218 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell1:SELECT=11,nova_cell1:INSERT=17 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:55:05.363496 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-15765aa7-9039-4d8f-80d5-296c68fdd811 tempest-VolumesAdminNegativeTest-1120895641 tempest-VolumesAdminNegativeTest-1120895641-project] Took 0.93 seconds to select destinations for 1 instance(s). {{(pid=120657) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 18 21:55:05.382643 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-15765aa7-9039-4d8f-80d5-296c68fdd811 tempest-VolumesAdminNegativeTest-1120895641 tempest-VolumesAdminNegativeTest-1120895641-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 21:55:05.383088 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-15765aa7-9039-4d8f-80d5-296c68fdd811 tempest-VolumesAdminNegativeTest-1120895641 tempest-VolumesAdminNegativeTest-1120895641-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 21:55:05.383470 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-15765aa7-9039-4d8f-80d5-296c68fdd811 tempest-VolumesAdminNegativeTest-1120895641 tempest-VolumesAdminNegativeTest-1120895641-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 21:55:05.569452 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-15765aa7-9039-4d8f-80d5-296c68fdd811 tempest-VolumesAdminNegativeTest-1120895641 tempest-VolumesAdminNegativeTest-1120895641-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 21:55:05.570058 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-15765aa7-9039-4d8f-80d5-296c68fdd811 tempest-VolumesAdminNegativeTest-1120895641 tempest-VolumesAdminNegativeTest-1120895641-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 21:55:05.570477 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-15765aa7-9039-4d8f-80d5-296c68fdd811 tempest-VolumesAdminNegativeTest-1120895641 tempest-VolumesAdminNegativeTest-1120895641-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 21:55:05.571139 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-15765aa7-9039-4d8f-80d5-296c68fdd811 tempest-VolumesAdminNegativeTest-1120895641 tempest-VolumesAdminNegativeTest-1120895641-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 21:55:05.571561 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-15765aa7-9039-4d8f-80d5-296c68fdd811 tempest-VolumesAdminNegativeTest-1120895641 tempest-VolumesAdminNegativeTest-1120895641-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 21:55:05.571992 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-15765aa7-9039-4d8f-80d5-296c68fdd811 tempest-VolumesAdminNegativeTest-1120895641 tempest-VolumesAdminNegativeTest-1120895641-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 21:55:05.588305 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-15765aa7-9039-4d8f-80d5-296c68fdd811 tempest-VolumesAdminNegativeTest-1120895641 tempest-VolumesAdminNegativeTest-1120895641-project] Getting quotas for project 00dd8fabfa51435e857fe7ad97ceaa87. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 18 21:55:05.593282 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-15765aa7-9039-4d8f-80d5-296c68fdd811 tempest-VolumesAdminNegativeTest-1120895641 tempest-VolumesAdminNegativeTest-1120895641-project] Getting quotas for user 4ef91c36862a4d84a8924f69141763cf and project 00dd8fabfa51435e857fe7ad97ceaa87. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 18 21:55:05.604225 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-15765aa7-9039-4d8f-80d5-296c68fdd811 tempest-VolumesAdminNegativeTest-1120895641 tempest-VolumesAdminNegativeTest-1120895641-project] [instance: 1db467ef-9831-4dbc-ab45-ec2e5d00d9fe] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120657) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 18 21:55:05.605735 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-15765aa7-9039-4d8f-80d5-296c68fdd811 tempest-VolumesAdminNegativeTest-1120895641 tempest-VolumesAdminNegativeTest-1120895641-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 21:55:05.606772 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-15765aa7-9039-4d8f-80d5-296c68fdd811 tempest-VolumesAdminNegativeTest-1120895641 tempest-VolumesAdminNegativeTest-1120895641-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 21:55:05.607178 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-15765aa7-9039-4d8f-80d5-296c68fdd811 tempest-VolumesAdminNegativeTest-1120895641 tempest-VolumesAdminNegativeTest-1120895641-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 21:55:05.695375 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-15765aa7-9039-4d8f-80d5-296c68fdd811 tempest-VolumesAdminNegativeTest-1120895641 tempest-VolumesAdminNegativeTest-1120895641-project] [instance: 1db467ef-9831-4dbc-ab45-ec2e5d00d9fe] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='87b0fec9-a948-41f9-a3c0-f0a6a5a8f1e7',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120657) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 18 21:55:05.696776 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-15765aa7-9039-4d8f-80d5-296c68fdd811 tempest-VolumesAdminNegativeTest-1120895641 tempest-VolumesAdminNegativeTest-1120895641-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 21:55:05.697290 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-15765aa7-9039-4d8f-80d5-296c68fdd811 tempest-VolumesAdminNegativeTest-1120895641 tempest-VolumesAdminNegativeTest-1120895641-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 21:55:05.697750 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-15765aa7-9039-4d8f-80d5-296c68fdd811 tempest-VolumesAdminNegativeTest-1120895641 tempest-VolumesAdminNegativeTest-1120895641-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 21:55:05.959245 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-15765aa7-9039-4d8f-80d5-296c68fdd811 tempest-VolumesAdminNegativeTest-1120895641 tempest-VolumesAdminNegativeTest-1120895641-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 21:55:05.960421 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-15765aa7-9039-4d8f-80d5-296c68fdd811 tempest-VolumesAdminNegativeTest-1120895641 tempest-VolumesAdminNegativeTest-1120895641-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 21:55:05.960931 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-15765aa7-9039-4d8f-80d5-296c68fdd811 tempest-VolumesAdminNegativeTest-1120895641 tempest-VolumesAdminNegativeTest-1120895641-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 21:55:05.961583 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-15765aa7-9039-4d8f-80d5-296c68fdd811 tempest-VolumesAdminNegativeTest-1120895641 tempest-VolumesAdminNegativeTest-1120895641-project] Acquiring lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 18 21:55:05.962062 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-15765aa7-9039-4d8f-80d5-296c68fdd811 tempest-VolumesAdminNegativeTest-1120895641 tempest-VolumesAdminNegativeTest-1120895641-project] Acquired lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 18 21:55:05.963001 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-3c1fda02-fec9-44e4-8d80-3973fb6a12f2 None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 21:55:05.963511 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-3c1fda02-fec9-44e4-8d80-3973fb6a12f2 None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 21:55:05.964019 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-3c1fda02-fec9-44e4-8d80-3973fb6a12f2 None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 21:55:05.964948 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-3c1fda02-fec9-44e4-8d80-3973fb6a12f2 None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 21:55:05.966338 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-3c1fda02-fec9-44e4-8d80-3973fb6a12f2 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 21:55:05.967083 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-3c1fda02-fec9-44e4-8d80-3973fb6a12f2 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 21:55:05.976872 n-d-853704-1 nova-conductor[120657]: ERROR nova.compute.rpcapi [None req-3c1fda02-fec9-44e4-8d80-3973fb6a12f2 None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 18 21:55:05.977361 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-3c1fda02-fec9-44e4-8d80-3973fb6a12f2 None None] Releasing lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 18 21:55:05.978494 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server [None req-3c1fda02-fec9-44e4-8d80-3973fb6a12f2 None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 21:55:05.978494 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 21:55:05.978494 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 18 21:55:05.978494 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 18 21:55:05.978494 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 18 21:55:05.978494 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 21:55:05.978494 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 18 21:55:05.978494 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 21:55:05.978494 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 21:55:05.978494 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 18 21:55:05.978494 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 18 21:55:05.978494 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 18 21:55:05.978494 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 18 21:55:05.978494 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 18 21:55:05.978494 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 18 21:55:05.978494 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 18 21:55:05.978494 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 18 21:55:05.985221 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 18 21:55:05.985221 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 18 21:55:05.985221 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 18 21:55:05.985221 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 18 21:55:05.985221 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 18 21:55:05.985221 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 18 21:55:05.985221 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 21:55:05.985221 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 21:55:13.896407 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=12,nova_api:UPDATE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:55:13.965151 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=12,nova_api:DELETE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:55:14.056291 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:55:14.057110 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell1:SELECT=11,nova_cell1:SAVEPOINT=1,nova_cell1:INSERT=18,nova_cell1:RELEASE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:55:15.792084 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=12 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:55:15.876307 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=23,nova_api:UPDATE=2,nova_api:DELETE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:55:15.976923 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:55:15.976923 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell1:SELECT=11,nova_cell1:SAVEPOINT=1,nova_cell1:INSERT=18,nova_cell1:RELEASE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:55:22.881892 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=20,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:55:42.926290 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=12,nova_cell0:UPDATE=3 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:55:44.346351 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-4bbae670-958f-4217-9ecd-adaff2311a23 tempest-ImagesOneServerTestJSON-1955418274 tempest-ImagesOneServerTestJSON-1955418274-project] Took 0.53 seconds to select destinations for 1 instance(s). {{(pid=120658) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 18 21:55:44.367118 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-4bbae670-958f-4217-9ecd-adaff2311a23 tempest-ImagesOneServerTestJSON-1955418274 tempest-ImagesOneServerTestJSON-1955418274-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 21:55:44.367570 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-4bbae670-958f-4217-9ecd-adaff2311a23 tempest-ImagesOneServerTestJSON-1955418274 tempest-ImagesOneServerTestJSON-1955418274-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 21:55:44.368006 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-4bbae670-958f-4217-9ecd-adaff2311a23 tempest-ImagesOneServerTestJSON-1955418274 tempest-ImagesOneServerTestJSON-1955418274-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 21:55:44.552699 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-4bbae670-958f-4217-9ecd-adaff2311a23 tempest-ImagesOneServerTestJSON-1955418274 tempest-ImagesOneServerTestJSON-1955418274-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 21:55:44.553144 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-4bbae670-958f-4217-9ecd-adaff2311a23 tempest-ImagesOneServerTestJSON-1955418274 tempest-ImagesOneServerTestJSON-1955418274-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 21:55:44.553541 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-4bbae670-958f-4217-9ecd-adaff2311a23 tempest-ImagesOneServerTestJSON-1955418274 tempest-ImagesOneServerTestJSON-1955418274-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 21:55:44.554206 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-4bbae670-958f-4217-9ecd-adaff2311a23 tempest-ImagesOneServerTestJSON-1955418274 tempest-ImagesOneServerTestJSON-1955418274-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 21:55:44.554621 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-4bbae670-958f-4217-9ecd-adaff2311a23 tempest-ImagesOneServerTestJSON-1955418274 tempest-ImagesOneServerTestJSON-1955418274-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 21:55:44.555008 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-4bbae670-958f-4217-9ecd-adaff2311a23 tempest-ImagesOneServerTestJSON-1955418274 tempest-ImagesOneServerTestJSON-1955418274-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 21:55:44.570412 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-4bbae670-958f-4217-9ecd-adaff2311a23 tempest-ImagesOneServerTestJSON-1955418274 tempest-ImagesOneServerTestJSON-1955418274-project] Getting quotas for project a44bf07d6b0546c7b26d35f343845037. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 18 21:55:44.574410 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-4bbae670-958f-4217-9ecd-adaff2311a23 tempest-ImagesOneServerTestJSON-1955418274 tempest-ImagesOneServerTestJSON-1955418274-project] Getting quotas for user ecd06ef737db42b9a2bc388e4c5808bf and project a44bf07d6b0546c7b26d35f343845037. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 18 21:55:44.584916 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-4bbae670-958f-4217-9ecd-adaff2311a23 tempest-ImagesOneServerTestJSON-1955418274 tempest-ImagesOneServerTestJSON-1955418274-project] [instance: 8058d0b6-a27d-4858-a104-3ab3bc9268a0] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120658) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 18 21:55:44.585681 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-4bbae670-958f-4217-9ecd-adaff2311a23 tempest-ImagesOneServerTestJSON-1955418274 tempest-ImagesOneServerTestJSON-1955418274-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 21:55:44.586116 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-4bbae670-958f-4217-9ecd-adaff2311a23 tempest-ImagesOneServerTestJSON-1955418274 tempest-ImagesOneServerTestJSON-1955418274-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 21:55:44.586511 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-4bbae670-958f-4217-9ecd-adaff2311a23 tempest-ImagesOneServerTestJSON-1955418274 tempest-ImagesOneServerTestJSON-1955418274-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 21:55:44.669938 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-4bbae670-958f-4217-9ecd-adaff2311a23 tempest-ImagesOneServerTestJSON-1955418274 tempest-ImagesOneServerTestJSON-1955418274-project] [instance: 8058d0b6-a27d-4858-a104-3ab3bc9268a0] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='87b0fec9-a948-41f9-a3c0-f0a6a5a8f1e7',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120658) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 18 21:55:44.671061 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-4bbae670-958f-4217-9ecd-adaff2311a23 tempest-ImagesOneServerTestJSON-1955418274 tempest-ImagesOneServerTestJSON-1955418274-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 21:55:44.671498 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-4bbae670-958f-4217-9ecd-adaff2311a23 tempest-ImagesOneServerTestJSON-1955418274 tempest-ImagesOneServerTestJSON-1955418274-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 21:55:44.671920 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-4bbae670-958f-4217-9ecd-adaff2311a23 tempest-ImagesOneServerTestJSON-1955418274 tempest-ImagesOneServerTestJSON-1955418274-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 21:55:44.912914 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-4bbae670-958f-4217-9ecd-adaff2311a23 tempest-ImagesOneServerTestJSON-1955418274 tempest-ImagesOneServerTestJSON-1955418274-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 21:55:44.913355 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-4bbae670-958f-4217-9ecd-adaff2311a23 tempest-ImagesOneServerTestJSON-1955418274 tempest-ImagesOneServerTestJSON-1955418274-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 21:55:44.913760 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-4bbae670-958f-4217-9ecd-adaff2311a23 tempest-ImagesOneServerTestJSON-1955418274 tempest-ImagesOneServerTestJSON-1955418274-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 21:55:44.914249 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-4bbae670-958f-4217-9ecd-adaff2311a23 tempest-ImagesOneServerTestJSON-1955418274 tempest-ImagesOneServerTestJSON-1955418274-project] Acquiring lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 18 21:55:44.914696 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-4bbae670-958f-4217-9ecd-adaff2311a23 tempest-ImagesOneServerTestJSON-1955418274 tempest-ImagesOneServerTestJSON-1955418274-project] Acquired lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 18 21:55:44.915466 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-7747bd8b-311b-40e9-8744-5107032cbdeb None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 21:55:44.915919 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-7747bd8b-311b-40e9-8744-5107032cbdeb None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 21:55:44.916303 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-7747bd8b-311b-40e9-8744-5107032cbdeb None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 21:55:44.916979 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-7747bd8b-311b-40e9-8744-5107032cbdeb None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 21:55:44.917383 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-7747bd8b-311b-40e9-8744-5107032cbdeb None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 21:55:44.917775 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-7747bd8b-311b-40e9-8744-5107032cbdeb None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 21:55:44.927601 n-d-853704-1 nova-conductor[120658]: ERROR nova.compute.rpcapi [None req-7747bd8b-311b-40e9-8744-5107032cbdeb None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 18 21:55:44.928119 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-7747bd8b-311b-40e9-8744-5107032cbdeb None None] Releasing lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 18 21:55:44.929130 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server [None req-7747bd8b-311b-40e9-8744-5107032cbdeb None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 21:55:44.929130 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 21:55:44.929130 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 18 21:55:44.929130 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 18 21:55:44.929130 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 18 21:55:44.929130 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 21:55:44.929130 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 18 21:55:44.929130 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 21:55:44.929130 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 21:55:44.929130 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 18 21:55:44.929130 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 18 21:55:44.929130 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 18 21:55:44.929130 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 18 21:55:44.929130 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 18 21:55:44.929130 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 18 21:55:44.929130 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 18 21:55:44.929130 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 18 21:55:44.930576 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 18 21:55:44.930576 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 18 21:55:44.930576 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 18 21:55:44.930576 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 18 21:55:44.930576 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 18 21:55:44.930576 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 18 21:55:44.930576 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 21:55:44.930576 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 21:55:54.817525 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=11,nova_api:UPDATE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:55:54.818275 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=13,nova_api:DELETE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:55:54.926972 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:55:54.927797 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell1:SELECT=11,nova_cell1:SAVEPOINT=1,nova_cell1:INSERT=18,nova_cell1:RELEASE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:56:02.872025 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:56:21.524519 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-bab7bfca-06f7-425b-a07b-b202f1072560 tempest-ServersTestMultiNic-773389532 tempest-ServersTestMultiNic-773389532-project] Took 0.57 seconds to select destinations for 1 instance(s). {{(pid=120657) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 18 21:56:21.546815 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-bab7bfca-06f7-425b-a07b-b202f1072560 tempest-ServersTestMultiNic-773389532 tempest-ServersTestMultiNic-773389532-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 21:56:21.550000 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-bab7bfca-06f7-425b-a07b-b202f1072560 tempest-ServersTestMultiNic-773389532 tempest-ServersTestMultiNic-773389532-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 21:56:21.550502 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-bab7bfca-06f7-425b-a07b-b202f1072560 tempest-ServersTestMultiNic-773389532 tempest-ServersTestMultiNic-773389532-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.004s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 21:56:21.657324 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-bab7bfca-06f7-425b-a07b-b202f1072560 tempest-ServersTestMultiNic-773389532 tempest-ServersTestMultiNic-773389532-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 21:56:21.657769 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-bab7bfca-06f7-425b-a07b-b202f1072560 tempest-ServersTestMultiNic-773389532 tempest-ServersTestMultiNic-773389532-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 21:56:21.658160 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-bab7bfca-06f7-425b-a07b-b202f1072560 tempest-ServersTestMultiNic-773389532 tempest-ServersTestMultiNic-773389532-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 21:56:21.658832 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-bab7bfca-06f7-425b-a07b-b202f1072560 tempest-ServersTestMultiNic-773389532 tempest-ServersTestMultiNic-773389532-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 21:56:21.659238 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-bab7bfca-06f7-425b-a07b-b202f1072560 tempest-ServersTestMultiNic-773389532 tempest-ServersTestMultiNic-773389532-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 21:56:21.659592 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-bab7bfca-06f7-425b-a07b-b202f1072560 tempest-ServersTestMultiNic-773389532 tempest-ServersTestMultiNic-773389532-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 21:56:21.675787 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-bab7bfca-06f7-425b-a07b-b202f1072560 tempest-ServersTestMultiNic-773389532 tempest-ServersTestMultiNic-773389532-project] Getting quotas for project 6c760d0117f747aeb13f8d85dad5c7a0. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 18 21:56:21.679801 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-bab7bfca-06f7-425b-a07b-b202f1072560 tempest-ServersTestMultiNic-773389532 tempest-ServersTestMultiNic-773389532-project] Getting quotas for user 7a09f989352e472697d48a8423113cdc and project 6c760d0117f747aeb13f8d85dad5c7a0. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 18 21:56:21.690576 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-bab7bfca-06f7-425b-a07b-b202f1072560 tempest-ServersTestMultiNic-773389532 tempest-ServersTestMultiNic-773389532-project] [instance: f40bc07b-894f-4eee-9f3c-fb71e73d9874] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120657) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 18 21:56:21.691345 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-bab7bfca-06f7-425b-a07b-b202f1072560 tempest-ServersTestMultiNic-773389532 tempest-ServersTestMultiNic-773389532-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 21:56:21.691821 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-bab7bfca-06f7-425b-a07b-b202f1072560 tempest-ServersTestMultiNic-773389532 tempest-ServersTestMultiNic-773389532-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 21:56:21.692213 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-bab7bfca-06f7-425b-a07b-b202f1072560 tempest-ServersTestMultiNic-773389532 tempest-ServersTestMultiNic-773389532-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 21:56:21.784153 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-bab7bfca-06f7-425b-a07b-b202f1072560 tempest-ServersTestMultiNic-773389532 tempest-ServersTestMultiNic-773389532-project] [instance: f40bc07b-894f-4eee-9f3c-fb71e73d9874] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='87b0fec9-a948-41f9-a3c0-f0a6a5a8f1e7',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120657) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 18 21:56:21.785683 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-bab7bfca-06f7-425b-a07b-b202f1072560 tempest-ServersTestMultiNic-773389532 tempest-ServersTestMultiNic-773389532-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 21:56:21.786223 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-bab7bfca-06f7-425b-a07b-b202f1072560 tempest-ServersTestMultiNic-773389532 tempest-ServersTestMultiNic-773389532-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 21:56:21.786869 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-bab7bfca-06f7-425b-a07b-b202f1072560 tempest-ServersTestMultiNic-773389532 tempest-ServersTestMultiNic-773389532-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 21:56:22.053159 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-bab7bfca-06f7-425b-a07b-b202f1072560 tempest-ServersTestMultiNic-773389532 tempest-ServersTestMultiNic-773389532-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 21:56:22.053597 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-bab7bfca-06f7-425b-a07b-b202f1072560 tempest-ServersTestMultiNic-773389532 tempest-ServersTestMultiNic-773389532-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 21:56:22.054007 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-bab7bfca-06f7-425b-a07b-b202f1072560 tempest-ServersTestMultiNic-773389532 tempest-ServersTestMultiNic-773389532-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 21:56:22.055098 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-bab7bfca-06f7-425b-a07b-b202f1072560 tempest-ServersTestMultiNic-773389532 tempest-ServersTestMultiNic-773389532-project] Acquiring lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 18 21:56:22.055503 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-bab7bfca-06f7-425b-a07b-b202f1072560 tempest-ServersTestMultiNic-773389532 tempest-ServersTestMultiNic-773389532-project] Acquired lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 18 21:56:22.056301 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-7d7da1f1-8615-497c-813c-5b933e0dd8dd None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 21:56:22.056735 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-7d7da1f1-8615-497c-813c-5b933e0dd8dd None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 21:56:22.057160 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-7d7da1f1-8615-497c-813c-5b933e0dd8dd None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 21:56:22.057818 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-7d7da1f1-8615-497c-813c-5b933e0dd8dd None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 21:56:22.058232 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-7d7da1f1-8615-497c-813c-5b933e0dd8dd None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 21:56:22.058733 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-7d7da1f1-8615-497c-813c-5b933e0dd8dd None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 21:56:22.070005 n-d-853704-1 nova-conductor[120657]: ERROR nova.compute.rpcapi [None req-7d7da1f1-8615-497c-813c-5b933e0dd8dd None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 18 21:56:22.074336 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-7d7da1f1-8615-497c-813c-5b933e0dd8dd None None] Releasing lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 18 21:56:22.074336 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server [None req-7d7da1f1-8615-497c-813c-5b933e0dd8dd None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 21:56:22.074336 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 21:56:22.074336 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 18 21:56:22.074336 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 18 21:56:22.074336 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 18 21:56:22.074336 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 21:56:22.074336 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 18 21:56:22.074336 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 21:56:22.074336 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 21:56:22.074336 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 18 21:56:22.074336 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 18 21:56:22.074336 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 18 21:56:22.074336 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 18 21:56:22.074336 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 18 21:56:22.074336 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 18 21:56:22.075915 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 18 21:56:22.075915 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 18 21:56:22.075915 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 18 21:56:22.075915 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 18 21:56:22.075915 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 18 21:56:22.075915 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 18 21:56:22.075915 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 18 21:56:22.075915 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 18 21:56:22.075915 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 21:56:22.075915 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 21:56:22.923731 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=14,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:56:31.951985 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=13 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:56:31.952687 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=11,nova_api:UPDATE=1,nova_api:DELETE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:56:32.068098 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:56:32.068853 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell1:SELECT=11,nova_cell1:INSERT=17 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:56:42.877972 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=14,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:56:42.928270 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=3,nova_cell0:UPDATE=2 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:57:22.889133 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:57:22.942685 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:58:02.898451 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=7,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:58:02.943431 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=3 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:58:21.177773 n-d-853704-1 nova-conductor[120658]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 18 21:58:22.900905 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=4,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:58:22.951845 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=3,nova_cell0:UPDATE=2 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:59:02.899071 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:59:02.952998 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:59:32.900411 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=5,nova_cell0:UPDATE=3 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 21:59:32.908127 n-d-853704-1 nova-conductor[120657]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 18 21:59:42.943615 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:00:12.901858 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:00:14.407336 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-ac5d22c1-22e9-4103-b6c4-12485aa599ab tempest-DeleteServersAdminTestJSON-327361073 tempest-DeleteServersAdminTestJSON-327361073-project] Took 0.46 seconds to select destinations for 1 instance(s). {{(pid=120658) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 18 22:00:14.431256 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ac5d22c1-22e9-4103-b6c4-12485aa599ab tempest-DeleteServersAdminTestJSON-327361073 tempest-DeleteServersAdminTestJSON-327361073-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:00:14.431734 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ac5d22c1-22e9-4103-b6c4-12485aa599ab tempest-DeleteServersAdminTestJSON-327361073 tempest-DeleteServersAdminTestJSON-327361073-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:00:14.432604 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ac5d22c1-22e9-4103-b6c4-12485aa599ab tempest-DeleteServersAdminTestJSON-327361073 tempest-DeleteServersAdminTestJSON-327361073-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:00:14.551201 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:00:14.553512 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ac5d22c1-22e9-4103-b6c4-12485aa599ab tempest-DeleteServersAdminTestJSON-327361073 tempest-DeleteServersAdminTestJSON-327361073-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:00:14.554191 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ac5d22c1-22e9-4103-b6c4-12485aa599ab tempest-DeleteServersAdminTestJSON-327361073 tempest-DeleteServersAdminTestJSON-327361073-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:00:14.554464 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ac5d22c1-22e9-4103-b6c4-12485aa599ab tempest-DeleteServersAdminTestJSON-327361073 tempest-DeleteServersAdminTestJSON-327361073-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:00:14.555185 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ac5d22c1-22e9-4103-b6c4-12485aa599ab tempest-DeleteServersAdminTestJSON-327361073 tempest-DeleteServersAdminTestJSON-327361073-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:00:14.555605 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ac5d22c1-22e9-4103-b6c4-12485aa599ab tempest-DeleteServersAdminTestJSON-327361073 tempest-DeleteServersAdminTestJSON-327361073-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:00:14.556024 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ac5d22c1-22e9-4103-b6c4-12485aa599ab tempest-DeleteServersAdminTestJSON-327361073 tempest-DeleteServersAdminTestJSON-327361073-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:00:14.572340 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-ac5d22c1-22e9-4103-b6c4-12485aa599ab tempest-DeleteServersAdminTestJSON-327361073 tempest-DeleteServersAdminTestJSON-327361073-project] Getting quotas for project 4538a3db5b634db3b684656d07abe905. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 18 22:00:14.577111 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-ac5d22c1-22e9-4103-b6c4-12485aa599ab tempest-DeleteServersAdminTestJSON-327361073 tempest-DeleteServersAdminTestJSON-327361073-project] Getting quotas for user 4e77bbbfeef840929431a8600eabbe5e and project 4538a3db5b634db3b684656d07abe905. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 18 22:00:14.587908 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-ac5d22c1-22e9-4103-b6c4-12485aa599ab tempest-DeleteServersAdminTestJSON-327361073 tempest-DeleteServersAdminTestJSON-327361073-project] [instance: cf7b7b8d-2b63-449c-a043-ffe4be201f2d] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120658) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 18 22:00:14.588680 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ac5d22c1-22e9-4103-b6c4-12485aa599ab tempest-DeleteServersAdminTestJSON-327361073 tempest-DeleteServersAdminTestJSON-327361073-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:00:14.589144 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ac5d22c1-22e9-4103-b6c4-12485aa599ab tempest-DeleteServersAdminTestJSON-327361073 tempest-DeleteServersAdminTestJSON-327361073-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:00:14.589504 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ac5d22c1-22e9-4103-b6c4-12485aa599ab tempest-DeleteServersAdminTestJSON-327361073 tempest-DeleteServersAdminTestJSON-327361073-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:00:14.663637 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-ac5d22c1-22e9-4103-b6c4-12485aa599ab tempest-DeleteServersAdminTestJSON-327361073 tempest-DeleteServersAdminTestJSON-327361073-project] [instance: cf7b7b8d-2b63-449c-a043-ffe4be201f2d] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='87b0fec9-a948-41f9-a3c0-f0a6a5a8f1e7',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120658) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 18 22:00:14.664997 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ac5d22c1-22e9-4103-b6c4-12485aa599ab tempest-DeleteServersAdminTestJSON-327361073 tempest-DeleteServersAdminTestJSON-327361073-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:00:14.665438 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ac5d22c1-22e9-4103-b6c4-12485aa599ab tempest-DeleteServersAdminTestJSON-327361073 tempest-DeleteServersAdminTestJSON-327361073-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:00:14.665835 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ac5d22c1-22e9-4103-b6c4-12485aa599ab tempest-DeleteServersAdminTestJSON-327361073 tempest-DeleteServersAdminTestJSON-327361073-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:00:14.898518 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ac5d22c1-22e9-4103-b6c4-12485aa599ab tempest-DeleteServersAdminTestJSON-327361073 tempest-DeleteServersAdminTestJSON-327361073-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:00:14.898981 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ac5d22c1-22e9-4103-b6c4-12485aa599ab tempest-DeleteServersAdminTestJSON-327361073 tempest-DeleteServersAdminTestJSON-327361073-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:00:14.899379 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ac5d22c1-22e9-4103-b6c4-12485aa599ab tempest-DeleteServersAdminTestJSON-327361073 tempest-DeleteServersAdminTestJSON-327361073-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:00:14.899878 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ac5d22c1-22e9-4103-b6c4-12485aa599ab tempest-DeleteServersAdminTestJSON-327361073 tempest-DeleteServersAdminTestJSON-327361073-project] Acquiring lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 18 22:00:14.900275 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ac5d22c1-22e9-4103-b6c4-12485aa599ab tempest-DeleteServersAdminTestJSON-327361073 tempest-DeleteServersAdminTestJSON-327361073-project] Acquired lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 18 22:00:14.901033 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-e6814534-f518-4c93-8668-e1162eccb26e None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:00:14.901449 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-e6814534-f518-4c93-8668-e1162eccb26e None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:00:14.901836 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-e6814534-f518-4c93-8668-e1162eccb26e None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:00:14.902485 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-e6814534-f518-4c93-8668-e1162eccb26e None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:00:14.902894 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-e6814534-f518-4c93-8668-e1162eccb26e None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:00:14.903900 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-e6814534-f518-4c93-8668-e1162eccb26e None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:00:14.912675 n-d-853704-1 nova-conductor[120658]: ERROR nova.compute.rpcapi [None req-e6814534-f518-4c93-8668-e1162eccb26e None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 18 22:00:14.913118 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-e6814534-f518-4c93-8668-e1162eccb26e None None] Releasing lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 18 22:00:14.914008 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server [None req-e6814534-f518-4c93-8668-e1162eccb26e None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:00:14.914008 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:00:14.914008 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 18 22:00:14.914008 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 18 22:00:14.914008 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 18 22:00:14.914008 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 22:00:14.914008 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 18 22:00:14.914008 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 22:00:14.914008 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:00:14.914008 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 18 22:00:14.914008 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 18 22:00:14.914008 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 18 22:00:14.914008 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 18 22:00:14.914008 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 18 22:00:14.914008 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 18 22:00:14.914008 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 18 22:00:14.914008 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 18 22:00:14.915460 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 18 22:00:14.915460 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 18 22:00:14.915460 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 18 22:00:14.915460 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 18 22:00:14.915460 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 18 22:00:14.915460 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 18 22:00:14.915460 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:00:14.915460 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 22:00:24.764569 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=12 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:00:24.825544 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=12,nova_api:UPDATE=1,nova_api:DELETE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:00:24.909472 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:00:24.910432 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell1:SELECT=11,nova_cell1:INSERT=17 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:00:29.134372 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-73b08162-e029-482d-a399-fa79db78f5fd tempest-LiveMigrationTest-1883958412 tempest-LiveMigrationTest-1883958412-project] Took 0.40 seconds to select destinations for 1 instance(s). {{(pid=120657) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 18 22:00:29.155212 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-73b08162-e029-482d-a399-fa79db78f5fd tempest-LiveMigrationTest-1883958412 tempest-LiveMigrationTest-1883958412-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:00:29.155656 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-73b08162-e029-482d-a399-fa79db78f5fd tempest-LiveMigrationTest-1883958412 tempest-LiveMigrationTest-1883958412-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:00:29.156079 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-73b08162-e029-482d-a399-fa79db78f5fd tempest-LiveMigrationTest-1883958412 tempest-LiveMigrationTest-1883958412-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:00:29.297844 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-73b08162-e029-482d-a399-fa79db78f5fd tempest-LiveMigrationTest-1883958412 tempest-LiveMigrationTest-1883958412-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:00:29.298320 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-73b08162-e029-482d-a399-fa79db78f5fd tempest-LiveMigrationTest-1883958412 tempest-LiveMigrationTest-1883958412-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:00:29.298723 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-73b08162-e029-482d-a399-fa79db78f5fd tempest-LiveMigrationTest-1883958412 tempest-LiveMigrationTest-1883958412-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:00:29.299392 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-73b08162-e029-482d-a399-fa79db78f5fd tempest-LiveMigrationTest-1883958412 tempest-LiveMigrationTest-1883958412-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:00:29.299858 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-73b08162-e029-482d-a399-fa79db78f5fd tempest-LiveMigrationTest-1883958412 tempest-LiveMigrationTest-1883958412-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:00:29.300282 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-73b08162-e029-482d-a399-fa79db78f5fd tempest-LiveMigrationTest-1883958412 tempest-LiveMigrationTest-1883958412-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:00:29.316685 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-73b08162-e029-482d-a399-fa79db78f5fd tempest-LiveMigrationTest-1883958412 tempest-LiveMigrationTest-1883958412-project] Getting quotas for project 29de488917ec43a09800ec6cb2187b56. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 18 22:00:29.320825 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-73b08162-e029-482d-a399-fa79db78f5fd tempest-LiveMigrationTest-1883958412 tempest-LiveMigrationTest-1883958412-project] Getting quotas for user c137b718cd2445909cacb17658bff8f6 and project 29de488917ec43a09800ec6cb2187b56. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 18 22:00:29.331646 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-73b08162-e029-482d-a399-fa79db78f5fd tempest-LiveMigrationTest-1883958412 tempest-LiveMigrationTest-1883958412-project] [instance: b145257d-a7bb-430f-86d2-4377be7605e9] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120657) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 18 22:00:29.332476 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-73b08162-e029-482d-a399-fa79db78f5fd tempest-LiveMigrationTest-1883958412 tempest-LiveMigrationTest-1883958412-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:00:29.332925 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-73b08162-e029-482d-a399-fa79db78f5fd tempest-LiveMigrationTest-1883958412 tempest-LiveMigrationTest-1883958412-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:00:29.333327 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-73b08162-e029-482d-a399-fa79db78f5fd tempest-LiveMigrationTest-1883958412 tempest-LiveMigrationTest-1883958412-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:00:29.405515 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-73b08162-e029-482d-a399-fa79db78f5fd tempest-LiveMigrationTest-1883958412 tempest-LiveMigrationTest-1883958412-project] [instance: b145257d-a7bb-430f-86d2-4377be7605e9] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='87b0fec9-a948-41f9-a3c0-f0a6a5a8f1e7',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120657) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 18 22:00:29.406974 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-73b08162-e029-482d-a399-fa79db78f5fd tempest-LiveMigrationTest-1883958412 tempest-LiveMigrationTest-1883958412-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:00:29.407421 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-73b08162-e029-482d-a399-fa79db78f5fd tempest-LiveMigrationTest-1883958412 tempest-LiveMigrationTest-1883958412-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:00:29.408514 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-73b08162-e029-482d-a399-fa79db78f5fd tempest-LiveMigrationTest-1883958412 tempest-LiveMigrationTest-1883958412-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:00:29.644282 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-73b08162-e029-482d-a399-fa79db78f5fd tempest-LiveMigrationTest-1883958412 tempest-LiveMigrationTest-1883958412-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:00:29.644651 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-73b08162-e029-482d-a399-fa79db78f5fd tempest-LiveMigrationTest-1883958412 tempest-LiveMigrationTest-1883958412-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:00:29.645050 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-73b08162-e029-482d-a399-fa79db78f5fd tempest-LiveMigrationTest-1883958412 tempest-LiveMigrationTest-1883958412-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:00:29.645506 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-73b08162-e029-482d-a399-fa79db78f5fd tempest-LiveMigrationTest-1883958412 tempest-LiveMigrationTest-1883958412-project] Acquiring lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 18 22:00:29.645868 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-73b08162-e029-482d-a399-fa79db78f5fd tempest-LiveMigrationTest-1883958412 tempest-LiveMigrationTest-1883958412-project] Acquired lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 18 22:00:29.646631 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-61c905b0-2ac9-4266-bf22-dfac7135c915 None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:00:29.647046 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-61c905b0-2ac9-4266-bf22-dfac7135c915 None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:00:29.647447 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-61c905b0-2ac9-4266-bf22-dfac7135c915 None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:00:29.648134 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-61c905b0-2ac9-4266-bf22-dfac7135c915 None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:00:29.648662 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-61c905b0-2ac9-4266-bf22-dfac7135c915 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:00:29.648991 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-61c905b0-2ac9-4266-bf22-dfac7135c915 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:00:29.658665 n-d-853704-1 nova-conductor[120657]: ERROR nova.compute.rpcapi [None req-61c905b0-2ac9-4266-bf22-dfac7135c915 None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 18 22:00:29.659064 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-61c905b0-2ac9-4266-bf22-dfac7135c915 None None] Releasing lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 18 22:00:29.660064 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server [None req-61c905b0-2ac9-4266-bf22-dfac7135c915 None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:00:29.660064 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:00:29.660064 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 18 22:00:29.660064 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 18 22:00:29.660064 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 18 22:00:29.660064 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 22:00:29.660064 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 18 22:00:29.660064 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 22:00:29.660064 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:00:29.660064 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 18 22:00:29.660064 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 18 22:00:29.660064 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 18 22:00:29.660064 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 18 22:00:29.660064 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 18 22:00:29.660064 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 18 22:00:29.660064 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 18 22:00:29.660064 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 18 22:00:29.662465 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 18 22:00:29.662465 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 18 22:00:29.662465 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 18 22:00:29.662465 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 18 22:00:29.662465 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 18 22:00:29.662465 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 18 22:00:29.662465 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:00:29.662465 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 22:00:39.579276 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=14,nova_api:UPDATE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:00:39.582550 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=10,nova_api:DELETE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:00:39.656227 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:00:39.656966 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell1:SELECT=11,nova_cell1:INSERT=17 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:00:52.900960 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=15,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:00:52.932585 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=12,nova_cell0:UPDATE=3 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:00:59.028309 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-933da3a0-1b33-4ff1-bfb1-59763b15a781 tempest-LiveAutoBlockMigrationV225Test-1869140755 tempest-LiveAutoBlockMigrationV225Test-1869140755-project] Took 0.45 seconds to select destinations for 1 instance(s). {{(pid=120658) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 18 22:00:59.065317 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-933da3a0-1b33-4ff1-bfb1-59763b15a781 tempest-LiveAutoBlockMigrationV225Test-1869140755 tempest-LiveAutoBlockMigrationV225Test-1869140755-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:00:59.065829 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-933da3a0-1b33-4ff1-bfb1-59763b15a781 tempest-LiveAutoBlockMigrationV225Test-1869140755 tempest-LiveAutoBlockMigrationV225Test-1869140755-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:00:59.066421 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-933da3a0-1b33-4ff1-bfb1-59763b15a781 tempest-LiveAutoBlockMigrationV225Test-1869140755 tempest-LiveAutoBlockMigrationV225Test-1869140755-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:00:59.322877 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-933da3a0-1b33-4ff1-bfb1-59763b15a781 tempest-LiveAutoBlockMigrationV225Test-1869140755 tempest-LiveAutoBlockMigrationV225Test-1869140755-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:00:59.324079 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-933da3a0-1b33-4ff1-bfb1-59763b15a781 tempest-LiveAutoBlockMigrationV225Test-1869140755 tempest-LiveAutoBlockMigrationV225Test-1869140755-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:00:59.324484 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-933da3a0-1b33-4ff1-bfb1-59763b15a781 tempest-LiveAutoBlockMigrationV225Test-1869140755 tempest-LiveAutoBlockMigrationV225Test-1869140755-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:00:59.325160 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-933da3a0-1b33-4ff1-bfb1-59763b15a781 tempest-LiveAutoBlockMigrationV225Test-1869140755 tempest-LiveAutoBlockMigrationV225Test-1869140755-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:00:59.325576 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-933da3a0-1b33-4ff1-bfb1-59763b15a781 tempest-LiveAutoBlockMigrationV225Test-1869140755 tempest-LiveAutoBlockMigrationV225Test-1869140755-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:00:59.325964 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-933da3a0-1b33-4ff1-bfb1-59763b15a781 tempest-LiveAutoBlockMigrationV225Test-1869140755 tempest-LiveAutoBlockMigrationV225Test-1869140755-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:00:59.342629 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-933da3a0-1b33-4ff1-bfb1-59763b15a781 tempest-LiveAutoBlockMigrationV225Test-1869140755 tempest-LiveAutoBlockMigrationV225Test-1869140755-project] Getting quotas for project 4c471d428ca241faa5d631d226526b3b. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 18 22:00:59.346742 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-933da3a0-1b33-4ff1-bfb1-59763b15a781 tempest-LiveAutoBlockMigrationV225Test-1869140755 tempest-LiveAutoBlockMigrationV225Test-1869140755-project] Getting quotas for user faee7152cc454a189fcbdd8f2d4b7b29 and project 4c471d428ca241faa5d631d226526b3b. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 18 22:00:59.357304 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-933da3a0-1b33-4ff1-bfb1-59763b15a781 tempest-LiveAutoBlockMigrationV225Test-1869140755 tempest-LiveAutoBlockMigrationV225Test-1869140755-project] [instance: 82b6b78e-3e35-4edf-ba05-9caa2bd0a1a6] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120658) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 18 22:00:59.358087 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-933da3a0-1b33-4ff1-bfb1-59763b15a781 tempest-LiveAutoBlockMigrationV225Test-1869140755 tempest-LiveAutoBlockMigrationV225Test-1869140755-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:00:59.358520 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-933da3a0-1b33-4ff1-bfb1-59763b15a781 tempest-LiveAutoBlockMigrationV225Test-1869140755 tempest-LiveAutoBlockMigrationV225Test-1869140755-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:00:59.358904 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-933da3a0-1b33-4ff1-bfb1-59763b15a781 tempest-LiveAutoBlockMigrationV225Test-1869140755 tempest-LiveAutoBlockMigrationV225Test-1869140755-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:00:59.460827 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-933da3a0-1b33-4ff1-bfb1-59763b15a781 tempest-LiveAutoBlockMigrationV225Test-1869140755 tempest-LiveAutoBlockMigrationV225Test-1869140755-project] [instance: 82b6b78e-3e35-4edf-ba05-9caa2bd0a1a6] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='87b0fec9-a948-41f9-a3c0-f0a6a5a8f1e7',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120658) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 18 22:00:59.460827 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-933da3a0-1b33-4ff1-bfb1-59763b15a781 tempest-LiveAutoBlockMigrationV225Test-1869140755 tempest-LiveAutoBlockMigrationV225Test-1869140755-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:00:59.460827 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-933da3a0-1b33-4ff1-bfb1-59763b15a781 tempest-LiveAutoBlockMigrationV225Test-1869140755 tempest-LiveAutoBlockMigrationV225Test-1869140755-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:00:59.461238 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-933da3a0-1b33-4ff1-bfb1-59763b15a781 tempest-LiveAutoBlockMigrationV225Test-1869140755 tempest-LiveAutoBlockMigrationV225Test-1869140755-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:00:59.720220 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-933da3a0-1b33-4ff1-bfb1-59763b15a781 tempest-LiveAutoBlockMigrationV225Test-1869140755 tempest-LiveAutoBlockMigrationV225Test-1869140755-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:00:59.720809 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-933da3a0-1b33-4ff1-bfb1-59763b15a781 tempest-LiveAutoBlockMigrationV225Test-1869140755 tempest-LiveAutoBlockMigrationV225Test-1869140755-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:00:59.721260 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-933da3a0-1b33-4ff1-bfb1-59763b15a781 tempest-LiveAutoBlockMigrationV225Test-1869140755 tempest-LiveAutoBlockMigrationV225Test-1869140755-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:00:59.721852 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-933da3a0-1b33-4ff1-bfb1-59763b15a781 tempest-LiveAutoBlockMigrationV225Test-1869140755 tempest-LiveAutoBlockMigrationV225Test-1869140755-project] Acquiring lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 18 22:00:59.722300 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-933da3a0-1b33-4ff1-bfb1-59763b15a781 tempest-LiveAutoBlockMigrationV225Test-1869140755 tempest-LiveAutoBlockMigrationV225Test-1869140755-project] Acquired lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 18 22:00:59.723256 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-834356fd-dd2c-447e-bb8c-fbf7f8cbfda0 None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:00:59.723764 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-834356fd-dd2c-447e-bb8c-fbf7f8cbfda0 None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:00:59.724221 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-834356fd-dd2c-447e-bb8c-fbf7f8cbfda0 None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:00:59.724950 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-834356fd-dd2c-447e-bb8c-fbf7f8cbfda0 None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:00:59.725430 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-834356fd-dd2c-447e-bb8c-fbf7f8cbfda0 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:00:59.725880 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-834356fd-dd2c-447e-bb8c-fbf7f8cbfda0 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:00:59.735840 n-d-853704-1 nova-conductor[120658]: ERROR nova.compute.rpcapi [None req-834356fd-dd2c-447e-bb8c-fbf7f8cbfda0 None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 18 22:00:59.736287 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-834356fd-dd2c-447e-bb8c-fbf7f8cbfda0 None None] Releasing lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 18 22:00:59.737280 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server [None req-834356fd-dd2c-447e-bb8c-fbf7f8cbfda0 None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:00:59.737280 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:00:59.737280 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 18 22:00:59.737280 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 18 22:00:59.737280 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 18 22:00:59.737280 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 22:00:59.737280 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 18 22:00:59.737280 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 22:00:59.737280 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:00:59.737280 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 18 22:00:59.737280 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 18 22:00:59.737280 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 18 22:00:59.737280 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 18 22:00:59.737280 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 18 22:00:59.737280 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 18 22:00:59.737280 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 18 22:00:59.737280 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 18 22:00:59.738843 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 18 22:00:59.738843 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 18 22:00:59.738843 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 18 22:00:59.738843 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 18 22:00:59.738843 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 18 22:00:59.738843 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 18 22:00:59.738843 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:00:59.738843 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 22:01:07.321457 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-d9f13713-9196-49aa-9f4e-51309b7ee22f tempest-ImagesTestJSON-1761366649 tempest-ImagesTestJSON-1761366649-project] Took 0.98 seconds to select destinations for 1 instance(s). {{(pid=120657) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 18 22:01:07.356714 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-d9f13713-9196-49aa-9f4e-51309b7ee22f tempest-ImagesTestJSON-1761366649 tempest-ImagesTestJSON-1761366649-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:01:07.357167 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-d9f13713-9196-49aa-9f4e-51309b7ee22f tempest-ImagesTestJSON-1761366649 tempest-ImagesTestJSON-1761366649-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:01:07.357604 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-d9f13713-9196-49aa-9f4e-51309b7ee22f tempest-ImagesTestJSON-1761366649 tempest-ImagesTestJSON-1761366649-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:01:07.586273 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-d9f13713-9196-49aa-9f4e-51309b7ee22f tempest-ImagesTestJSON-1761366649 tempest-ImagesTestJSON-1761366649-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:01:07.586995 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-d9f13713-9196-49aa-9f4e-51309b7ee22f tempest-ImagesTestJSON-1761366649 tempest-ImagesTestJSON-1761366649-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:01:07.587574 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-d9f13713-9196-49aa-9f4e-51309b7ee22f tempest-ImagesTestJSON-1761366649 tempest-ImagesTestJSON-1761366649-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:01:07.588409 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-d9f13713-9196-49aa-9f4e-51309b7ee22f tempest-ImagesTestJSON-1761366649 tempest-ImagesTestJSON-1761366649-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:01:07.589014 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-d9f13713-9196-49aa-9f4e-51309b7ee22f tempest-ImagesTestJSON-1761366649 tempest-ImagesTestJSON-1761366649-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:01:07.589411 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-d9f13713-9196-49aa-9f4e-51309b7ee22f tempest-ImagesTestJSON-1761366649 tempest-ImagesTestJSON-1761366649-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:01:07.608635 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-d9f13713-9196-49aa-9f4e-51309b7ee22f tempest-ImagesTestJSON-1761366649 tempest-ImagesTestJSON-1761366649-project] Getting quotas for project 677821f62eb848bcaf47c31a724218df. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 18 22:01:07.616941 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-d9f13713-9196-49aa-9f4e-51309b7ee22f tempest-ImagesTestJSON-1761366649 tempest-ImagesTestJSON-1761366649-project] Getting quotas for user 2b6fddb708684c6a8d541e617aa2fc48 and project 677821f62eb848bcaf47c31a724218df. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 18 22:01:07.630152 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-d9f13713-9196-49aa-9f4e-51309b7ee22f tempest-ImagesTestJSON-1761366649 tempest-ImagesTestJSON-1761366649-project] [instance: 6a93f456-9389-4297-99d7-4ff7aa76cdd2] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120657) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 18 22:01:07.631208 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-d9f13713-9196-49aa-9f4e-51309b7ee22f tempest-ImagesTestJSON-1761366649 tempest-ImagesTestJSON-1761366649-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:01:07.631640 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-d9f13713-9196-49aa-9f4e-51309b7ee22f tempest-ImagesTestJSON-1761366649 tempest-ImagesTestJSON-1761366649-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:01:07.632065 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-d9f13713-9196-49aa-9f4e-51309b7ee22f tempest-ImagesTestJSON-1761366649 tempest-ImagesTestJSON-1761366649-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:01:07.764732 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-d9f13713-9196-49aa-9f4e-51309b7ee22f tempest-ImagesTestJSON-1761366649 tempest-ImagesTestJSON-1761366649-project] [instance: 6a93f456-9389-4297-99d7-4ff7aa76cdd2] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='87b0fec9-a948-41f9-a3c0-f0a6a5a8f1e7',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120657) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 18 22:01:07.766617 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-d9f13713-9196-49aa-9f4e-51309b7ee22f tempest-ImagesTestJSON-1761366649 tempest-ImagesTestJSON-1761366649-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:01:07.767128 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-d9f13713-9196-49aa-9f4e-51309b7ee22f tempest-ImagesTestJSON-1761366649 tempest-ImagesTestJSON-1761366649-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:01:07.767592 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-d9f13713-9196-49aa-9f4e-51309b7ee22f tempest-ImagesTestJSON-1761366649 tempest-ImagesTestJSON-1761366649-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:01:08.114438 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-d9f13713-9196-49aa-9f4e-51309b7ee22f tempest-ImagesTestJSON-1761366649 tempest-ImagesTestJSON-1761366649-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:01:08.114880 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-d9f13713-9196-49aa-9f4e-51309b7ee22f tempest-ImagesTestJSON-1761366649 tempest-ImagesTestJSON-1761366649-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:01:08.115282 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-d9f13713-9196-49aa-9f4e-51309b7ee22f tempest-ImagesTestJSON-1761366649 tempest-ImagesTestJSON-1761366649-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:01:08.116152 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-d9f13713-9196-49aa-9f4e-51309b7ee22f tempest-ImagesTestJSON-1761366649 tempest-ImagesTestJSON-1761366649-project] Acquiring lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 18 22:01:08.116653 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-d9f13713-9196-49aa-9f4e-51309b7ee22f tempest-ImagesTestJSON-1761366649 tempest-ImagesTestJSON-1761366649-project] Acquired lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 18 22:01:08.118287 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-59b39ef6-3cd9-4132-98a0-389ab7b1ea62 None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:01:08.118590 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-59b39ef6-3cd9-4132-98a0-389ab7b1ea62 None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:01:08.119014 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-59b39ef6-3cd9-4132-98a0-389ab7b1ea62 None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:01:08.120546 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-59b39ef6-3cd9-4132-98a0-389ab7b1ea62 None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:01:08.121080 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-59b39ef6-3cd9-4132-98a0-389ab7b1ea62 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:01:08.121471 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-59b39ef6-3cd9-4132-98a0-389ab7b1ea62 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:01:08.130531 n-d-853704-1 nova-conductor[120657]: ERROR nova.compute.rpcapi [None req-59b39ef6-3cd9-4132-98a0-389ab7b1ea62 None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 18 22:01:08.130920 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-59b39ef6-3cd9-4132-98a0-389ab7b1ea62 None None] Releasing lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 18 22:01:08.131852 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server [None req-59b39ef6-3cd9-4132-98a0-389ab7b1ea62 None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:01:08.131852 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:01:08.131852 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 18 22:01:08.131852 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 18 22:01:08.131852 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 18 22:01:08.131852 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 22:01:08.131852 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 18 22:01:08.131852 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 22:01:08.131852 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:01:08.131852 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 18 22:01:08.131852 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 18 22:01:08.131852 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 18 22:01:08.131852 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 18 22:01:08.131852 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 18 22:01:08.131852 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 18 22:01:08.131852 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 18 22:01:08.131852 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 18 22:01:08.133180 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 18 22:01:08.133180 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 18 22:01:08.133180 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 18 22:01:08.133180 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 18 22:01:08.133180 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 18 22:01:08.133180 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 18 22:01:08.133180 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:01:08.133180 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 22:01:09.640468 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=13,nova_api:UPDATE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:01:09.641697 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=11,nova_api:DELETE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:01:09.733816 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:01:09.733816 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell1:SELECT=11,nova_cell1:INSERT=17 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:01:10.148406 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-14103436-9bb6-45ad-a1a3-582aa1fa017f tempest-VolumesAdminNegativeTest-1120895641 tempest-VolumesAdminNegativeTest-1120895641-project] Took 0.45 seconds to select destinations for 1 instance(s). {{(pid=120658) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 18 22:01:10.173332 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-14103436-9bb6-45ad-a1a3-582aa1fa017f tempest-VolumesAdminNegativeTest-1120895641 tempest-VolumesAdminNegativeTest-1120895641-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:01:10.174059 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-14103436-9bb6-45ad-a1a3-582aa1fa017f tempest-VolumesAdminNegativeTest-1120895641 tempest-VolumesAdminNegativeTest-1120895641-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:01:10.174646 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-14103436-9bb6-45ad-a1a3-582aa1fa017f tempest-VolumesAdminNegativeTest-1120895641 tempest-VolumesAdminNegativeTest-1120895641-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:01:10.317531 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-14103436-9bb6-45ad-a1a3-582aa1fa017f tempest-VolumesAdminNegativeTest-1120895641 tempest-VolumesAdminNegativeTest-1120895641-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:01:10.318068 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-14103436-9bb6-45ad-a1a3-582aa1fa017f tempest-VolumesAdminNegativeTest-1120895641 tempest-VolumesAdminNegativeTest-1120895641-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:01:10.318539 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-14103436-9bb6-45ad-a1a3-582aa1fa017f tempest-VolumesAdminNegativeTest-1120895641 tempest-VolumesAdminNegativeTest-1120895641-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:01:10.319272 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-14103436-9bb6-45ad-a1a3-582aa1fa017f tempest-VolumesAdminNegativeTest-1120895641 tempest-VolumesAdminNegativeTest-1120895641-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:01:10.319863 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-14103436-9bb6-45ad-a1a3-582aa1fa017f tempest-VolumesAdminNegativeTest-1120895641 tempest-VolumesAdminNegativeTest-1120895641-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:01:10.320495 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-14103436-9bb6-45ad-a1a3-582aa1fa017f tempest-VolumesAdminNegativeTest-1120895641 tempest-VolumesAdminNegativeTest-1120895641-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:01:10.336990 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-14103436-9bb6-45ad-a1a3-582aa1fa017f tempest-VolumesAdminNegativeTest-1120895641 tempest-VolumesAdminNegativeTest-1120895641-project] Getting quotas for project 00dd8fabfa51435e857fe7ad97ceaa87. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 18 22:01:10.341927 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-14103436-9bb6-45ad-a1a3-582aa1fa017f tempest-VolumesAdminNegativeTest-1120895641 tempest-VolumesAdminNegativeTest-1120895641-project] Getting quotas for user 4ef91c36862a4d84a8924f69141763cf and project 00dd8fabfa51435e857fe7ad97ceaa87. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 18 22:01:10.353152 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-14103436-9bb6-45ad-a1a3-582aa1fa017f tempest-VolumesAdminNegativeTest-1120895641 tempest-VolumesAdminNegativeTest-1120895641-project] [instance: 74acdf1d-c938-482c-a504-1c5a0fedc6cb] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120658) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 18 22:01:10.353938 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-14103436-9bb6-45ad-a1a3-582aa1fa017f tempest-VolumesAdminNegativeTest-1120895641 tempest-VolumesAdminNegativeTest-1120895641-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:01:10.354374 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-14103436-9bb6-45ad-a1a3-582aa1fa017f tempest-VolumesAdminNegativeTest-1120895641 tempest-VolumesAdminNegativeTest-1120895641-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:01:10.354772 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-14103436-9bb6-45ad-a1a3-582aa1fa017f tempest-VolumesAdminNegativeTest-1120895641 tempest-VolumesAdminNegativeTest-1120895641-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:01:10.426853 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-14103436-9bb6-45ad-a1a3-582aa1fa017f tempest-VolumesAdminNegativeTest-1120895641 tempest-VolumesAdminNegativeTest-1120895641-project] [instance: 74acdf1d-c938-482c-a504-1c5a0fedc6cb] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='87b0fec9-a948-41f9-a3c0-f0a6a5a8f1e7',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120658) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 18 22:01:10.428172 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-14103436-9bb6-45ad-a1a3-582aa1fa017f tempest-VolumesAdminNegativeTest-1120895641 tempest-VolumesAdminNegativeTest-1120895641-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:01:10.428615 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-14103436-9bb6-45ad-a1a3-582aa1fa017f tempest-VolumesAdminNegativeTest-1120895641 tempest-VolumesAdminNegativeTest-1120895641-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:01:10.429045 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-14103436-9bb6-45ad-a1a3-582aa1fa017f tempest-VolumesAdminNegativeTest-1120895641 tempest-VolumesAdminNegativeTest-1120895641-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:01:10.651318 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-14103436-9bb6-45ad-a1a3-582aa1fa017f tempest-VolumesAdminNegativeTest-1120895641 tempest-VolumesAdminNegativeTest-1120895641-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:01:10.652028 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-14103436-9bb6-45ad-a1a3-582aa1fa017f tempest-VolumesAdminNegativeTest-1120895641 tempest-VolumesAdminNegativeTest-1120895641-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:01:10.652449 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-14103436-9bb6-45ad-a1a3-582aa1fa017f tempest-VolumesAdminNegativeTest-1120895641 tempest-VolumesAdminNegativeTest-1120895641-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:01:10.652939 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-14103436-9bb6-45ad-a1a3-582aa1fa017f tempest-VolumesAdminNegativeTest-1120895641 tempest-VolumesAdminNegativeTest-1120895641-project] Acquiring lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 18 22:01:10.653469 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-14103436-9bb6-45ad-a1a3-582aa1fa017f tempest-VolumesAdminNegativeTest-1120895641 tempest-VolumesAdminNegativeTest-1120895641-project] Acquired lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 18 22:01:10.654811 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-cac4d2d8-d21b-444a-b515-a323573fb111 None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:01:10.654811 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-cac4d2d8-d21b-444a-b515-a323573fb111 None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:01:10.655079 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-cac4d2d8-d21b-444a-b515-a323573fb111 None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:01:10.655712 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-cac4d2d8-d21b-444a-b515-a323573fb111 None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:01:10.659919 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-cac4d2d8-d21b-444a-b515-a323573fb111 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:01:10.659919 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-cac4d2d8-d21b-444a-b515-a323573fb111 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:01:10.666494 n-d-853704-1 nova-conductor[120658]: ERROR nova.compute.rpcapi [None req-cac4d2d8-d21b-444a-b515-a323573fb111 None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 18 22:01:10.666897 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-cac4d2d8-d21b-444a-b515-a323573fb111 None None] Releasing lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 18 22:01:10.667841 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server [None req-cac4d2d8-d21b-444a-b515-a323573fb111 None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:01:10.667841 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:01:10.667841 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 18 22:01:10.667841 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 18 22:01:10.667841 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 18 22:01:10.667841 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 22:01:10.667841 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 18 22:01:10.667841 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 22:01:10.667841 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:01:10.667841 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 18 22:01:10.667841 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 18 22:01:10.667841 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 18 22:01:10.667841 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 18 22:01:10.667841 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 18 22:01:10.667841 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 18 22:01:10.667841 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 18 22:01:10.667841 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 18 22:01:10.669388 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 18 22:01:10.669388 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 18 22:01:10.669388 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 18 22:01:10.669388 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 18 22:01:10.669388 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 18 22:01:10.669388 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 18 22:01:10.669388 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:01:10.669388 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 22:01:11.467846 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-3e6d9379-15f1-4c0c-a0fa-341fba605ece tempest-ServersOnMultiNodesTest-678731844 tempest-ServersOnMultiNodesTest-678731844-project] Took 0.54 seconds to select destinations for 1 instance(s). {{(pid=120657) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 18 22:01:11.487940 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-3e6d9379-15f1-4c0c-a0fa-341fba605ece tempest-ServersOnMultiNodesTest-678731844 tempest-ServersOnMultiNodesTest-678731844-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:01:11.488394 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-3e6d9379-15f1-4c0c-a0fa-341fba605ece tempest-ServersOnMultiNodesTest-678731844 tempest-ServersOnMultiNodesTest-678731844-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:01:11.488800 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-3e6d9379-15f1-4c0c-a0fa-341fba605ece tempest-ServersOnMultiNodesTest-678731844 tempest-ServersOnMultiNodesTest-678731844-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:01:11.729899 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-3e6d9379-15f1-4c0c-a0fa-341fba605ece tempest-ServersOnMultiNodesTest-678731844 tempest-ServersOnMultiNodesTest-678731844-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:01:11.730347 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-3e6d9379-15f1-4c0c-a0fa-341fba605ece tempest-ServersOnMultiNodesTest-678731844 tempest-ServersOnMultiNodesTest-678731844-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:01:11.730742 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-3e6d9379-15f1-4c0c-a0fa-341fba605ece tempest-ServersOnMultiNodesTest-678731844 tempest-ServersOnMultiNodesTest-678731844-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:01:11.731398 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-3e6d9379-15f1-4c0c-a0fa-341fba605ece tempest-ServersOnMultiNodesTest-678731844 tempest-ServersOnMultiNodesTest-678731844-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:01:11.732030 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-3e6d9379-15f1-4c0c-a0fa-341fba605ece tempest-ServersOnMultiNodesTest-678731844 tempest-ServersOnMultiNodesTest-678731844-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:01:11.732415 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-3e6d9379-15f1-4c0c-a0fa-341fba605ece tempest-ServersOnMultiNodesTest-678731844 tempest-ServersOnMultiNodesTest-678731844-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:01:11.748208 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-3e6d9379-15f1-4c0c-a0fa-341fba605ece tempest-ServersOnMultiNodesTest-678731844 tempest-ServersOnMultiNodesTest-678731844-project] Getting quotas for project aa0bb46f671947669fdc51a9a9e9be0c. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 18 22:01:11.752356 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-3e6d9379-15f1-4c0c-a0fa-341fba605ece tempest-ServersOnMultiNodesTest-678731844 tempest-ServersOnMultiNodesTest-678731844-project] Getting quotas for user 4674a7961ba54e0ea09a08833bd8021e and project aa0bb46f671947669fdc51a9a9e9be0c. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 18 22:01:11.763094 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-3e6d9379-15f1-4c0c-a0fa-341fba605ece tempest-ServersOnMultiNodesTest-678731844 tempest-ServersOnMultiNodesTest-678731844-project] [instance: 2ee62227-b05a-4057-8192-290af6bf67eb] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120657) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 18 22:01:11.763895 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-3e6d9379-15f1-4c0c-a0fa-341fba605ece tempest-ServersOnMultiNodesTest-678731844 tempest-ServersOnMultiNodesTest-678731844-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:01:11.764328 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-3e6d9379-15f1-4c0c-a0fa-341fba605ece tempest-ServersOnMultiNodesTest-678731844 tempest-ServersOnMultiNodesTest-678731844-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:01:11.764728 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-3e6d9379-15f1-4c0c-a0fa-341fba605ece tempest-ServersOnMultiNodesTest-678731844 tempest-ServersOnMultiNodesTest-678731844-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:01:11.867167 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-3e6d9379-15f1-4c0c-a0fa-341fba605ece tempest-ServersOnMultiNodesTest-678731844 tempest-ServersOnMultiNodesTest-678731844-project] [instance: 2ee62227-b05a-4057-8192-290af6bf67eb] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='87b0fec9-a948-41f9-a3c0-f0a6a5a8f1e7',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120657) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 18 22:01:11.868408 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-3e6d9379-15f1-4c0c-a0fa-341fba605ece tempest-ServersOnMultiNodesTest-678731844 tempest-ServersOnMultiNodesTest-678731844-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:01:11.868856 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-3e6d9379-15f1-4c0c-a0fa-341fba605ece tempest-ServersOnMultiNodesTest-678731844 tempest-ServersOnMultiNodesTest-678731844-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:01:11.869278 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-3e6d9379-15f1-4c0c-a0fa-341fba605ece tempest-ServersOnMultiNodesTest-678731844 tempest-ServersOnMultiNodesTest-678731844-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:01:12.197610 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-3e6d9379-15f1-4c0c-a0fa-341fba605ece tempest-ServersOnMultiNodesTest-678731844 tempest-ServersOnMultiNodesTest-678731844-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:01:12.198361 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-3e6d9379-15f1-4c0c-a0fa-341fba605ece tempest-ServersOnMultiNodesTest-678731844 tempest-ServersOnMultiNodesTest-678731844-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:01:12.198773 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-3e6d9379-15f1-4c0c-a0fa-341fba605ece tempest-ServersOnMultiNodesTest-678731844 tempest-ServersOnMultiNodesTest-678731844-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:01:12.199228 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-3e6d9379-15f1-4c0c-a0fa-341fba605ece tempest-ServersOnMultiNodesTest-678731844 tempest-ServersOnMultiNodesTest-678731844-project] Acquiring lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 18 22:01:12.199589 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-3e6d9379-15f1-4c0c-a0fa-341fba605ece tempest-ServersOnMultiNodesTest-678731844 tempest-ServersOnMultiNodesTest-678731844-project] Acquired lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 18 22:01:12.202923 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-be3771e1-f37a-4543-92fd-16b43e2226a2 None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:01:12.202923 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-be3771e1-f37a-4543-92fd-16b43e2226a2 None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:01:12.202923 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-be3771e1-f37a-4543-92fd-16b43e2226a2 None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:01:12.202923 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-be3771e1-f37a-4543-92fd-16b43e2226a2 None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:01:12.202923 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-be3771e1-f37a-4543-92fd-16b43e2226a2 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:01:12.203606 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-be3771e1-f37a-4543-92fd-16b43e2226a2 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:01:12.211037 n-d-853704-1 nova-conductor[120657]: ERROR nova.compute.rpcapi [None req-be3771e1-f37a-4543-92fd-16b43e2226a2 None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 18 22:01:12.211185 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-be3771e1-f37a-4543-92fd-16b43e2226a2 None None] Releasing lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 18 22:01:12.212114 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server [None req-be3771e1-f37a-4543-92fd-16b43e2226a2 None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:01:12.212114 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:01:12.212114 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 18 22:01:12.212114 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 18 22:01:12.212114 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 18 22:01:12.212114 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 22:01:12.212114 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 18 22:01:12.212114 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 22:01:12.212114 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:01:12.212114 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 18 22:01:12.212114 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 18 22:01:12.212114 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 18 22:01:12.212114 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 18 22:01:12.212114 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 18 22:01:12.212114 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 18 22:01:12.212114 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 18 22:01:12.212114 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 18 22:01:12.214261 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 18 22:01:12.214261 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 18 22:01:12.214261 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 18 22:01:12.214261 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 18 22:01:12.214261 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 18 22:01:12.214261 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 18 22:01:12.214261 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:01:12.214261 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 22:01:20.513370 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=13 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:01:20.595959 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=11,nova_api:UPDATE=1,nova_api:DELETE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:01:20.666478 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:01:20.667181 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell1:SELECT=11,nova_cell1:INSERT=17 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:01:22.097776 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=29 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:01:22.098566 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=19,nova_api:UPDATE=2,nova_api:DELETE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:01:22.209377 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=8 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:01:22.210230 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell1:SELECT=22,nova_cell1:SAVEPOINT=2,nova_cell1:INSERT=36,nova_cell1:RELEASE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:01:32.902026 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=20,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:01:32.942595 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=20,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:01:59.437555 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-88058bda-0631-4844-92d1-937a07f21b5f tempest-InstanceActionsTestJSON-1822072562 tempest-InstanceActionsTestJSON-1822072562-project] Took 0.60 seconds to select destinations for 1 instance(s). {{(pid=120658) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 18 22:01:59.459356 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-88058bda-0631-4844-92d1-937a07f21b5f tempest-InstanceActionsTestJSON-1822072562 tempest-InstanceActionsTestJSON-1822072562-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:01:59.460041 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-88058bda-0631-4844-92d1-937a07f21b5f tempest-InstanceActionsTestJSON-1822072562 tempest-InstanceActionsTestJSON-1822072562-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:01:59.460862 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-88058bda-0631-4844-92d1-937a07f21b5f tempest-InstanceActionsTestJSON-1822072562 tempest-InstanceActionsTestJSON-1822072562-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:01:59.683181 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-88058bda-0631-4844-92d1-937a07f21b5f tempest-InstanceActionsTestJSON-1822072562 tempest-InstanceActionsTestJSON-1822072562-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:01:59.683623 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-88058bda-0631-4844-92d1-937a07f21b5f tempest-InstanceActionsTestJSON-1822072562 tempest-InstanceActionsTestJSON-1822072562-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:01:59.684056 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-88058bda-0631-4844-92d1-937a07f21b5f tempest-InstanceActionsTestJSON-1822072562 tempest-InstanceActionsTestJSON-1822072562-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:01:59.684723 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-88058bda-0631-4844-92d1-937a07f21b5f tempest-InstanceActionsTestJSON-1822072562 tempest-InstanceActionsTestJSON-1822072562-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:01:59.685178 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-88058bda-0631-4844-92d1-937a07f21b5f tempest-InstanceActionsTestJSON-1822072562 tempest-InstanceActionsTestJSON-1822072562-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:01:59.685530 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-88058bda-0631-4844-92d1-937a07f21b5f tempest-InstanceActionsTestJSON-1822072562 tempest-InstanceActionsTestJSON-1822072562-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:01:59.702420 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-88058bda-0631-4844-92d1-937a07f21b5f tempest-InstanceActionsTestJSON-1822072562 tempest-InstanceActionsTestJSON-1822072562-project] Getting quotas for project a69e70054fb645baaba8d6affd78f496. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 18 22:01:59.706783 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-88058bda-0631-4844-92d1-937a07f21b5f tempest-InstanceActionsTestJSON-1822072562 tempest-InstanceActionsTestJSON-1822072562-project] Getting quotas for user cd603afb73e74d8a831acf0b16747beb and project a69e70054fb645baaba8d6affd78f496. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 18 22:01:59.718201 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-88058bda-0631-4844-92d1-937a07f21b5f tempest-InstanceActionsTestJSON-1822072562 tempest-InstanceActionsTestJSON-1822072562-project] [instance: f55c7002-a3c4-47fe-849e-e8781def070a] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120658) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 18 22:01:59.725052 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-88058bda-0631-4844-92d1-937a07f21b5f tempest-InstanceActionsTestJSON-1822072562 tempest-InstanceActionsTestJSON-1822072562-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:01:59.725052 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-88058bda-0631-4844-92d1-937a07f21b5f tempest-InstanceActionsTestJSON-1822072562 tempest-InstanceActionsTestJSON-1822072562-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:01:59.725052 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-88058bda-0631-4844-92d1-937a07f21b5f tempest-InstanceActionsTestJSON-1822072562 tempest-InstanceActionsTestJSON-1822072562-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:01:59.842562 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-88058bda-0631-4844-92d1-937a07f21b5f tempest-InstanceActionsTestJSON-1822072562 tempest-InstanceActionsTestJSON-1822072562-project] [instance: f55c7002-a3c4-47fe-849e-e8781def070a] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='87b0fec9-a948-41f9-a3c0-f0a6a5a8f1e7',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120658) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 18 22:01:59.844696 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-88058bda-0631-4844-92d1-937a07f21b5f tempest-InstanceActionsTestJSON-1822072562 tempest-InstanceActionsTestJSON-1822072562-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:01:59.845136 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-88058bda-0631-4844-92d1-937a07f21b5f tempest-InstanceActionsTestJSON-1822072562 tempest-InstanceActionsTestJSON-1822072562-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:01:59.845533 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-88058bda-0631-4844-92d1-937a07f21b5f tempest-InstanceActionsTestJSON-1822072562 tempest-InstanceActionsTestJSON-1822072562-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:02:00.150117 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-88058bda-0631-4844-92d1-937a07f21b5f tempest-InstanceActionsTestJSON-1822072562 tempest-InstanceActionsTestJSON-1822072562-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:02:00.150830 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-88058bda-0631-4844-92d1-937a07f21b5f tempest-InstanceActionsTestJSON-1822072562 tempest-InstanceActionsTestJSON-1822072562-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:02:00.151623 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-88058bda-0631-4844-92d1-937a07f21b5f tempest-InstanceActionsTestJSON-1822072562 tempest-InstanceActionsTestJSON-1822072562-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:02:00.152115 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-88058bda-0631-4844-92d1-937a07f21b5f tempest-InstanceActionsTestJSON-1822072562 tempest-InstanceActionsTestJSON-1822072562-project] Acquiring lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 18 22:02:00.152857 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-88058bda-0631-4844-92d1-937a07f21b5f tempest-InstanceActionsTestJSON-1822072562 tempest-InstanceActionsTestJSON-1822072562-project] Acquired lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 18 22:02:00.153200 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-0910396c-1d18-4654-a014-8ab17cd4d9ab None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:02:00.154866 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-0910396c-1d18-4654-a014-8ab17cd4d9ab None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:02:00.154866 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-0910396c-1d18-4654-a014-8ab17cd4d9ab None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:02:00.155072 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-0910396c-1d18-4654-a014-8ab17cd4d9ab None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:02:00.155497 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-0910396c-1d18-4654-a014-8ab17cd4d9ab None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:02:00.155915 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-0910396c-1d18-4654-a014-8ab17cd4d9ab None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:02:00.164930 n-d-853704-1 nova-conductor[120658]: ERROR nova.compute.rpcapi [None req-0910396c-1d18-4654-a014-8ab17cd4d9ab None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 18 22:02:00.165319 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-0910396c-1d18-4654-a014-8ab17cd4d9ab None None] Releasing lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 18 22:02:00.167045 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server [None req-0910396c-1d18-4654-a014-8ab17cd4d9ab None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:02:00.167045 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:02:00.167045 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 18 22:02:00.167045 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 18 22:02:00.167045 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 18 22:02:00.167045 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 22:02:00.167045 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 18 22:02:00.167045 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 22:02:00.167045 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:02:00.167045 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 18 22:02:00.167045 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 18 22:02:00.167045 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 18 22:02:00.167045 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 18 22:02:00.167045 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 18 22:02:00.167045 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 18 22:02:00.167045 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 18 22:02:00.167045 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 18 22:02:00.171491 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 18 22:02:00.171491 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 18 22:02:00.171491 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 18 22:02:00.171491 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 18 22:02:00.171491 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 18 22:02:00.171491 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 18 22:02:00.171491 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:02:00.171491 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 22:02:09.938904 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=14 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:02:10.038161 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=10,nova_api:UPDATE=1,nova_api:DELETE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:02:10.161826 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:02:10.163638 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell1:SELECT=11,nova_cell1:SAVEPOINT=1,nova_cell1:INSERT=18,nova_cell1:RELEASE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:02:12.898300 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:02:12.949810 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=13,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:02:38.881793 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-3842d40c-0e4d-4013-9539-fe3d6c3387b0 tempest-ServerDiskConfigTestJSON-1657751511 tempest-ServerDiskConfigTestJSON-1657751511-project] Took 0.57 seconds to select destinations for 1 instance(s). {{(pid=120657) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 18 22:02:38.904986 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-3842d40c-0e4d-4013-9539-fe3d6c3387b0 tempest-ServerDiskConfigTestJSON-1657751511 tempest-ServerDiskConfigTestJSON-1657751511-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:02:38.905527 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-3842d40c-0e4d-4013-9539-fe3d6c3387b0 tempest-ServerDiskConfigTestJSON-1657751511 tempest-ServerDiskConfigTestJSON-1657751511-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:02:38.908292 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-3842d40c-0e4d-4013-9539-fe3d6c3387b0 tempest-ServerDiskConfigTestJSON-1657751511 tempest-ServerDiskConfigTestJSON-1657751511-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:02:39.109402 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-3842d40c-0e4d-4013-9539-fe3d6c3387b0 tempest-ServerDiskConfigTestJSON-1657751511 tempest-ServerDiskConfigTestJSON-1657751511-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:02:39.109846 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-3842d40c-0e4d-4013-9539-fe3d6c3387b0 tempest-ServerDiskConfigTestJSON-1657751511 tempest-ServerDiskConfigTestJSON-1657751511-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:02:39.110241 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-3842d40c-0e4d-4013-9539-fe3d6c3387b0 tempest-ServerDiskConfigTestJSON-1657751511 tempest-ServerDiskConfigTestJSON-1657751511-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:02:39.110912 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-3842d40c-0e4d-4013-9539-fe3d6c3387b0 tempest-ServerDiskConfigTestJSON-1657751511 tempest-ServerDiskConfigTestJSON-1657751511-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:02:39.111326 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-3842d40c-0e4d-4013-9539-fe3d6c3387b0 tempest-ServerDiskConfigTestJSON-1657751511 tempest-ServerDiskConfigTestJSON-1657751511-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:02:39.111750 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-3842d40c-0e4d-4013-9539-fe3d6c3387b0 tempest-ServerDiskConfigTestJSON-1657751511 tempest-ServerDiskConfigTestJSON-1657751511-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:02:39.128518 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-3842d40c-0e4d-4013-9539-fe3d6c3387b0 tempest-ServerDiskConfigTestJSON-1657751511 tempest-ServerDiskConfigTestJSON-1657751511-project] Getting quotas for project 4450d6767020433e986d18bb2965de51. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 18 22:02:39.132937 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-3842d40c-0e4d-4013-9539-fe3d6c3387b0 tempest-ServerDiskConfigTestJSON-1657751511 tempest-ServerDiskConfigTestJSON-1657751511-project] Getting quotas for user 9a6f17cb5745439882594505d7f20df1 and project 4450d6767020433e986d18bb2965de51. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 18 22:02:39.143507 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-3842d40c-0e4d-4013-9539-fe3d6c3387b0 tempest-ServerDiskConfigTestJSON-1657751511 tempest-ServerDiskConfigTestJSON-1657751511-project] [instance: 03dcdf70-80fb-470e-a5b6-f33c40f1a8f9] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120657) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 18 22:02:39.144278 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-3842d40c-0e4d-4013-9539-fe3d6c3387b0 tempest-ServerDiskConfigTestJSON-1657751511 tempest-ServerDiskConfigTestJSON-1657751511-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:02:39.144723 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-3842d40c-0e4d-4013-9539-fe3d6c3387b0 tempest-ServerDiskConfigTestJSON-1657751511 tempest-ServerDiskConfigTestJSON-1657751511-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:02:39.145119 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-3842d40c-0e4d-4013-9539-fe3d6c3387b0 tempest-ServerDiskConfigTestJSON-1657751511 tempest-ServerDiskConfigTestJSON-1657751511-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:02:39.228976 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-3842d40c-0e4d-4013-9539-fe3d6c3387b0 tempest-ServerDiskConfigTestJSON-1657751511 tempest-ServerDiskConfigTestJSON-1657751511-project] [instance: 03dcdf70-80fb-470e-a5b6-f33c40f1a8f9] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='87b0fec9-a948-41f9-a3c0-f0a6a5a8f1e7',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120657) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 18 22:02:39.230421 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-3842d40c-0e4d-4013-9539-fe3d6c3387b0 tempest-ServerDiskConfigTestJSON-1657751511 tempest-ServerDiskConfigTestJSON-1657751511-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:02:39.231140 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-3842d40c-0e4d-4013-9539-fe3d6c3387b0 tempest-ServerDiskConfigTestJSON-1657751511 tempest-ServerDiskConfigTestJSON-1657751511-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:02:39.231842 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-3842d40c-0e4d-4013-9539-fe3d6c3387b0 tempest-ServerDiskConfigTestJSON-1657751511 tempest-ServerDiskConfigTestJSON-1657751511-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:02:39.470292 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-3842d40c-0e4d-4013-9539-fe3d6c3387b0 tempest-ServerDiskConfigTestJSON-1657751511 tempest-ServerDiskConfigTestJSON-1657751511-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:02:39.470649 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-3842d40c-0e4d-4013-9539-fe3d6c3387b0 tempest-ServerDiskConfigTestJSON-1657751511 tempest-ServerDiskConfigTestJSON-1657751511-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:02:39.471044 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-3842d40c-0e4d-4013-9539-fe3d6c3387b0 tempest-ServerDiskConfigTestJSON-1657751511 tempest-ServerDiskConfigTestJSON-1657751511-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:02:39.471477 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-3842d40c-0e4d-4013-9539-fe3d6c3387b0 tempest-ServerDiskConfigTestJSON-1657751511 tempest-ServerDiskConfigTestJSON-1657751511-project] Acquiring lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 18 22:02:39.471934 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-3842d40c-0e4d-4013-9539-fe3d6c3387b0 tempest-ServerDiskConfigTestJSON-1657751511 tempest-ServerDiskConfigTestJSON-1657751511-project] Acquired lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 18 22:02:39.472988 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c38c5e62-0913-44dd-ac83-dfeaca9e329e None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:02:39.473461 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c38c5e62-0913-44dd-ac83-dfeaca9e329e None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:02:39.473841 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c38c5e62-0913-44dd-ac83-dfeaca9e329e None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:02:39.474509 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c38c5e62-0913-44dd-ac83-dfeaca9e329e None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:02:39.474915 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c38c5e62-0913-44dd-ac83-dfeaca9e329e None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:02:39.475302 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c38c5e62-0913-44dd-ac83-dfeaca9e329e None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:02:39.485546 n-d-853704-1 nova-conductor[120657]: ERROR nova.compute.rpcapi [None req-c38c5e62-0913-44dd-ac83-dfeaca9e329e None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 18 22:02:39.485949 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c38c5e62-0913-44dd-ac83-dfeaca9e329e None None] Releasing lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 18 22:02:39.486875 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server [None req-c38c5e62-0913-44dd-ac83-dfeaca9e329e None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:02:39.486875 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:02:39.486875 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 18 22:02:39.486875 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 18 22:02:39.486875 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 18 22:02:39.486875 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 22:02:39.486875 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 18 22:02:39.486875 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 22:02:39.486875 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:02:39.486875 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 18 22:02:39.486875 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 18 22:02:39.486875 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 18 22:02:39.486875 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 18 22:02:39.486875 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 18 22:02:39.486875 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 18 22:02:39.486875 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 18 22:02:39.486875 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 18 22:02:39.488099 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 18 22:02:39.488099 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 18 22:02:39.488099 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 18 22:02:39.488099 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 18 22:02:39.488099 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 18 22:02:39.488099 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 18 22:02:39.488099 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:02:39.488099 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 22:02:49.386544 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=11,nova_api:UPDATE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:02:49.387269 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=13,nova_api:DELETE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:02:49.488060 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:02:49.491677 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell1:SELECT=11,nova_cell1:SAVEPOINT=1,nova_cell1:INSERT=18,nova_cell1:RELEASE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:02:52.899246 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=14,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:02:52.948069 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:03:12.955488 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=3,nova_cell0:UPDATE=2 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:03:32.905523 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:03:52.949634 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=3 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:04:12.907476 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=7,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:04:12.958367 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=3,nova_cell0:UPDATE=2 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:04:52.904380 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:04:52.954711 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:05:01.201989 n-d-853704-1 nova-conductor[120658]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 18 22:05:12.913700 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=3,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:05:32.955382 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=3 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:05:52.910214 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:06:12.910733 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=3,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:06:12.950157 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:06:32.921284 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=4,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:06:32.959084 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=3,nova_cell0:UPDATE=2 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:06:43.373338 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-5fd4841c-20dd-4632-92ea-cbca310881cf tempest-LiveMigrationNegativeTest-1198353743 tempest-LiveMigrationNegativeTest-1198353743-project] Took 0.84 seconds to select destinations for 1 instance(s). {{(pid=120658) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 18 22:06:43.395036 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-5fd4841c-20dd-4632-92ea-cbca310881cf tempest-LiveMigrationNegativeTest-1198353743 tempest-LiveMigrationNegativeTest-1198353743-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:06:43.395430 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-5fd4841c-20dd-4632-92ea-cbca310881cf tempest-LiveMigrationNegativeTest-1198353743 tempest-LiveMigrationNegativeTest-1198353743-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:06:43.395855 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-5fd4841c-20dd-4632-92ea-cbca310881cf tempest-LiveMigrationNegativeTest-1198353743 tempest-LiveMigrationNegativeTest-1198353743-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:06:43.611316 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-5fd4841c-20dd-4632-92ea-cbca310881cf tempest-LiveMigrationNegativeTest-1198353743 tempest-LiveMigrationNegativeTest-1198353743-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:06:43.611702 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-5fd4841c-20dd-4632-92ea-cbca310881cf tempest-LiveMigrationNegativeTest-1198353743 tempest-LiveMigrationNegativeTest-1198353743-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:06:43.612342 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-5fd4841c-20dd-4632-92ea-cbca310881cf tempest-LiveMigrationNegativeTest-1198353743 tempest-LiveMigrationNegativeTest-1198353743-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:06:43.613016 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-5fd4841c-20dd-4632-92ea-cbca310881cf tempest-LiveMigrationNegativeTest-1198353743 tempest-LiveMigrationNegativeTest-1198353743-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:06:43.613618 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-5fd4841c-20dd-4632-92ea-cbca310881cf tempest-LiveMigrationNegativeTest-1198353743 tempest-LiveMigrationNegativeTest-1198353743-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:06:43.614016 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-5fd4841c-20dd-4632-92ea-cbca310881cf tempest-LiveMigrationNegativeTest-1198353743 tempest-LiveMigrationNegativeTest-1198353743-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:06:43.629704 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-5fd4841c-20dd-4632-92ea-cbca310881cf tempest-LiveMigrationNegativeTest-1198353743 tempest-LiveMigrationNegativeTest-1198353743-project] Getting quotas for project 5e5533ee845b4a1290b8fc32a3030755. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 18 22:06:43.634278 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-5fd4841c-20dd-4632-92ea-cbca310881cf tempest-LiveMigrationNegativeTest-1198353743 tempest-LiveMigrationNegativeTest-1198353743-project] Getting quotas for user cbf8eb9046654b68bd8be43f6570cee9 and project 5e5533ee845b4a1290b8fc32a3030755. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 18 22:06:43.645199 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-5fd4841c-20dd-4632-92ea-cbca310881cf tempest-LiveMigrationNegativeTest-1198353743 tempest-LiveMigrationNegativeTest-1198353743-project] [instance: af13d6c0-c781-452a-9331-193fda13ef8a] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120658) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 18 22:06:43.646201 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-5fd4841c-20dd-4632-92ea-cbca310881cf tempest-LiveMigrationNegativeTest-1198353743 tempest-LiveMigrationNegativeTest-1198353743-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:06:43.647300 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-5fd4841c-20dd-4632-92ea-cbca310881cf tempest-LiveMigrationNegativeTest-1198353743 tempest-LiveMigrationNegativeTest-1198353743-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:06:43.647726 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-5fd4841c-20dd-4632-92ea-cbca310881cf tempest-LiveMigrationNegativeTest-1198353743 tempest-LiveMigrationNegativeTest-1198353743-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:06:43.687966 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-5fd4841c-20dd-4632-92ea-cbca310881cf tempest-LiveMigrationNegativeTest-1198353743 tempest-LiveMigrationNegativeTest-1198353743-project] [instance: af13d6c0-c781-452a-9331-193fda13ef8a] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='87b0fec9-a948-41f9-a3c0-f0a6a5a8f1e7',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120658) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 18 22:06:43.689514 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-5fd4841c-20dd-4632-92ea-cbca310881cf tempest-LiveMigrationNegativeTest-1198353743 tempest-LiveMigrationNegativeTest-1198353743-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:06:43.690184 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-5fd4841c-20dd-4632-92ea-cbca310881cf tempest-LiveMigrationNegativeTest-1198353743 tempest-LiveMigrationNegativeTest-1198353743-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:06:43.690819 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-5fd4841c-20dd-4632-92ea-cbca310881cf tempest-LiveMigrationNegativeTest-1198353743 tempest-LiveMigrationNegativeTest-1198353743-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:06:44.009114 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-5fd4841c-20dd-4632-92ea-cbca310881cf tempest-LiveMigrationNegativeTest-1198353743 tempest-LiveMigrationNegativeTest-1198353743-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:06:44.009890 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-5fd4841c-20dd-4632-92ea-cbca310881cf tempest-LiveMigrationNegativeTest-1198353743 tempest-LiveMigrationNegativeTest-1198353743-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:06:44.010496 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-5fd4841c-20dd-4632-92ea-cbca310881cf tempest-LiveMigrationNegativeTest-1198353743 tempest-LiveMigrationNegativeTest-1198353743-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:06:44.011214 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-5fd4841c-20dd-4632-92ea-cbca310881cf tempest-LiveMigrationNegativeTest-1198353743 tempest-LiveMigrationNegativeTest-1198353743-project] Acquiring lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 18 22:06:44.011587 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-5fd4841c-20dd-4632-92ea-cbca310881cf tempest-LiveMigrationNegativeTest-1198353743 tempest-LiveMigrationNegativeTest-1198353743-project] Acquired lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 18 22:06:44.012484 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-8ff1bbad-3e1a-49d1-922a-918022281440 None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:06:44.012923 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-8ff1bbad-3e1a-49d1-922a-918022281440 None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:06:44.022862 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-8ff1bbad-3e1a-49d1-922a-918022281440 None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:06:44.022862 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-8ff1bbad-3e1a-49d1-922a-918022281440 None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:06:44.022862 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-8ff1bbad-3e1a-49d1-922a-918022281440 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:06:44.022862 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-8ff1bbad-3e1a-49d1-922a-918022281440 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:06:44.024494 n-d-853704-1 nova-conductor[120658]: ERROR nova.compute.rpcapi [None req-8ff1bbad-3e1a-49d1-922a-918022281440 None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 18 22:06:44.024897 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-8ff1bbad-3e1a-49d1-922a-918022281440 None None] Releasing lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 18 22:06:44.026166 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server [None req-8ff1bbad-3e1a-49d1-922a-918022281440 None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:06:44.026166 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:06:44.026166 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 18 22:06:44.026166 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 18 22:06:44.026166 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 18 22:06:44.026166 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 22:06:44.026166 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 18 22:06:44.026166 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 22:06:44.026166 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:06:44.026166 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 18 22:06:44.026166 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 18 22:06:44.026166 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 18 22:06:44.026166 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 18 22:06:44.026166 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 18 22:06:44.026166 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 18 22:06:44.026166 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 18 22:06:44.026166 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 18 22:06:44.027586 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 18 22:06:44.027586 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 18 22:06:44.027586 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 18 22:06:44.027586 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 18 22:06:44.027586 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 18 22:06:44.027586 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 18 22:06:44.027586 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:06:44.027586 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 22:06:53.760841 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=11,nova_api:UPDATE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:06:53.885524 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=13,nova_api:DELETE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:06:54.022836 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:06:54.024412 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell1:SELECT=11,nova_cell1:SAVEPOINT=1,nova_cell1:INSERT=18,nova_cell1:RELEASE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:07:12.928958 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:07:12.968185 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=14,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:07:27.553963 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-43621c0c-cb3b-483a-9eff-c88ddbd5486b tempest-ImagesTestJSON-1761366649 tempest-ImagesTestJSON-1761366649-project] Took 0.42 seconds to select destinations for 1 instance(s). {{(pid=120657) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 18 22:07:27.577792 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-43621c0c-cb3b-483a-9eff-c88ddbd5486b tempest-ImagesTestJSON-1761366649 tempest-ImagesTestJSON-1761366649-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:07:27.578232 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-43621c0c-cb3b-483a-9eff-c88ddbd5486b tempest-ImagesTestJSON-1761366649 tempest-ImagesTestJSON-1761366649-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:07:27.578716 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-43621c0c-cb3b-483a-9eff-c88ddbd5486b tempest-ImagesTestJSON-1761366649 tempest-ImagesTestJSON-1761366649-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:07:27.705825 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-43621c0c-cb3b-483a-9eff-c88ddbd5486b tempest-ImagesTestJSON-1761366649 tempest-ImagesTestJSON-1761366649-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:07:27.706925 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-43621c0c-cb3b-483a-9eff-c88ddbd5486b tempest-ImagesTestJSON-1761366649 tempest-ImagesTestJSON-1761366649-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.002s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:07:27.707929 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-43621c0c-cb3b-483a-9eff-c88ddbd5486b tempest-ImagesTestJSON-1761366649 tempest-ImagesTestJSON-1761366649-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:07:27.709789 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-43621c0c-cb3b-483a-9eff-c88ddbd5486b tempest-ImagesTestJSON-1761366649 tempest-ImagesTestJSON-1761366649-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:07:27.710766 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-43621c0c-cb3b-483a-9eff-c88ddbd5486b tempest-ImagesTestJSON-1761366649 tempest-ImagesTestJSON-1761366649-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:07:27.712294 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-43621c0c-cb3b-483a-9eff-c88ddbd5486b tempest-ImagesTestJSON-1761366649 tempest-ImagesTestJSON-1761366649-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:07:27.733494 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-43621c0c-cb3b-483a-9eff-c88ddbd5486b tempest-ImagesTestJSON-1761366649 tempest-ImagesTestJSON-1761366649-project] Getting quotas for project 677821f62eb848bcaf47c31a724218df. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 18 22:07:27.737565 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-43621c0c-cb3b-483a-9eff-c88ddbd5486b tempest-ImagesTestJSON-1761366649 tempest-ImagesTestJSON-1761366649-project] Getting quotas for user 2b6fddb708684c6a8d541e617aa2fc48 and project 677821f62eb848bcaf47c31a724218df. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 18 22:07:27.761939 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-43621c0c-cb3b-483a-9eff-c88ddbd5486b tempest-ImagesTestJSON-1761366649 tempest-ImagesTestJSON-1761366649-project] [instance: f071615c-2b3f-429d-b8d5-0ba74732b3db] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120657) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 18 22:07:27.762767 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-43621c0c-cb3b-483a-9eff-c88ddbd5486b tempest-ImagesTestJSON-1761366649 tempest-ImagesTestJSON-1761366649-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:07:27.763209 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-43621c0c-cb3b-483a-9eff-c88ddbd5486b tempest-ImagesTestJSON-1761366649 tempest-ImagesTestJSON-1761366649-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:07:27.763874 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-43621c0c-cb3b-483a-9eff-c88ddbd5486b tempest-ImagesTestJSON-1761366649 tempest-ImagesTestJSON-1761366649-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:07:27.831483 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-43621c0c-cb3b-483a-9eff-c88ddbd5486b tempest-ImagesTestJSON-1761366649 tempest-ImagesTestJSON-1761366649-project] [instance: f071615c-2b3f-429d-b8d5-0ba74732b3db] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='87b0fec9-a948-41f9-a3c0-f0a6a5a8f1e7',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120657) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 18 22:07:27.832746 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-43621c0c-cb3b-483a-9eff-c88ddbd5486b tempest-ImagesTestJSON-1761366649 tempest-ImagesTestJSON-1761366649-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:07:27.833212 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-43621c0c-cb3b-483a-9eff-c88ddbd5486b tempest-ImagesTestJSON-1761366649 tempest-ImagesTestJSON-1761366649-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:07:27.833607 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-43621c0c-cb3b-483a-9eff-c88ddbd5486b tempest-ImagesTestJSON-1761366649 tempest-ImagesTestJSON-1761366649-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:07:28.386783 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-43621c0c-cb3b-483a-9eff-c88ddbd5486b tempest-ImagesTestJSON-1761366649 tempest-ImagesTestJSON-1761366649-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:07:28.387229 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-43621c0c-cb3b-483a-9eff-c88ddbd5486b tempest-ImagesTestJSON-1761366649 tempest-ImagesTestJSON-1761366649-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:07:28.387631 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-43621c0c-cb3b-483a-9eff-c88ddbd5486b tempest-ImagesTestJSON-1761366649 tempest-ImagesTestJSON-1761366649-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:07:28.388112 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-43621c0c-cb3b-483a-9eff-c88ddbd5486b tempest-ImagesTestJSON-1761366649 tempest-ImagesTestJSON-1761366649-project] Acquiring lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 18 22:07:28.388466 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-43621c0c-cb3b-483a-9eff-c88ddbd5486b tempest-ImagesTestJSON-1761366649 tempest-ImagesTestJSON-1761366649-project] Acquired lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 18 22:07:28.389191 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-df7bb9bb-f791-4235-8288-f876137578c2 None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:07:28.389619 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-df7bb9bb-f791-4235-8288-f876137578c2 None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:07:28.390078 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-df7bb9bb-f791-4235-8288-f876137578c2 None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:07:28.390710 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-df7bb9bb-f791-4235-8288-f876137578c2 None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:07:28.391119 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-df7bb9bb-f791-4235-8288-f876137578c2 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:07:28.391493 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-df7bb9bb-f791-4235-8288-f876137578c2 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:07:28.401601 n-d-853704-1 nova-conductor[120657]: ERROR nova.compute.rpcapi [None req-df7bb9bb-f791-4235-8288-f876137578c2 None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 18 22:07:28.402046 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-df7bb9bb-f791-4235-8288-f876137578c2 None None] Releasing lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 18 22:07:28.403221 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server [None req-df7bb9bb-f791-4235-8288-f876137578c2 None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:07:28.403221 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:07:28.403221 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 18 22:07:28.403221 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 18 22:07:28.403221 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 18 22:07:28.403221 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 22:07:28.403221 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 18 22:07:28.403221 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 22:07:28.403221 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:07:28.403221 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 18 22:07:28.403221 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 18 22:07:28.403221 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 18 22:07:28.403221 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 18 22:07:28.403221 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 18 22:07:28.403221 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 18 22:07:28.403221 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 18 22:07:28.403221 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 18 22:07:28.404559 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 18 22:07:28.404559 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 18 22:07:28.404559 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 18 22:07:28.404559 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 18 22:07:28.404559 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 18 22:07:28.404559 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 18 22:07:28.404559 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:07:28.404559 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 22:07:38.128296 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=10,nova_api:UPDATE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:07:38.234369 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=14,nova_api:DELETE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:07:38.400269 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:07:38.400912 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell1:SELECT=11,nova_cell1:INSERT=17 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:07:42.934930 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=11,nova_cell0:UPDATE=3 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:07:52.272124 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-0f60979f-d710-44f9-a406-b19280787ed3 tempest-FloatingIPsAssociationNegativeTestJSON-1553180457 tempest-FloatingIPsAssociationNegativeTestJSON-1553180457-project] Took 0.73 seconds to select destinations for 1 instance(s). {{(pid=120658) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 18 22:07:52.320655 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-0f60979f-d710-44f9-a406-b19280787ed3 tempest-FloatingIPsAssociationNegativeTestJSON-1553180457 tempest-FloatingIPsAssociationNegativeTestJSON-1553180457-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:07:52.321391 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-0f60979f-d710-44f9-a406-b19280787ed3 tempest-FloatingIPsAssociationNegativeTestJSON-1553180457 tempest-FloatingIPsAssociationNegativeTestJSON-1553180457-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:07:52.321895 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-0f60979f-d710-44f9-a406-b19280787ed3 tempest-FloatingIPsAssociationNegativeTestJSON-1553180457 tempest-FloatingIPsAssociationNegativeTestJSON-1553180457-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:07:52.638841 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:07:52.641480 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-0f60979f-d710-44f9-a406-b19280787ed3 tempest-FloatingIPsAssociationNegativeTestJSON-1553180457 tempest-FloatingIPsAssociationNegativeTestJSON-1553180457-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:07:52.642413 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-0f60979f-d710-44f9-a406-b19280787ed3 tempest-FloatingIPsAssociationNegativeTestJSON-1553180457 tempest-FloatingIPsAssociationNegativeTestJSON-1553180457-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:07:52.643080 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-0f60979f-d710-44f9-a406-b19280787ed3 tempest-FloatingIPsAssociationNegativeTestJSON-1553180457 tempest-FloatingIPsAssociationNegativeTestJSON-1553180457-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:07:52.644086 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-0f60979f-d710-44f9-a406-b19280787ed3 tempest-FloatingIPsAssociationNegativeTestJSON-1553180457 tempest-FloatingIPsAssociationNegativeTestJSON-1553180457-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:07:52.644757 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-0f60979f-d710-44f9-a406-b19280787ed3 tempest-FloatingIPsAssociationNegativeTestJSON-1553180457 tempest-FloatingIPsAssociationNegativeTestJSON-1553180457-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:07:52.645368 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-0f60979f-d710-44f9-a406-b19280787ed3 tempest-FloatingIPsAssociationNegativeTestJSON-1553180457 tempest-FloatingIPsAssociationNegativeTestJSON-1553180457-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:07:52.664470 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-0f60979f-d710-44f9-a406-b19280787ed3 tempest-FloatingIPsAssociationNegativeTestJSON-1553180457 tempest-FloatingIPsAssociationNegativeTestJSON-1553180457-project] Getting quotas for project 43671c3278d9405bbad46c4d2377f194. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 18 22:07:52.669441 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-0f60979f-d710-44f9-a406-b19280787ed3 tempest-FloatingIPsAssociationNegativeTestJSON-1553180457 tempest-FloatingIPsAssociationNegativeTestJSON-1553180457-project] Getting quotas for user 504be7043f6f452da2d455633bed9071 and project 43671c3278d9405bbad46c4d2377f194. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 18 22:07:52.689621 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-0f60979f-d710-44f9-a406-b19280787ed3 tempest-FloatingIPsAssociationNegativeTestJSON-1553180457 tempest-FloatingIPsAssociationNegativeTestJSON-1553180457-project] [instance: f4ad0f65-5439-4fd9-9d2d-5abda464364a] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120658) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 18 22:07:52.690439 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-0f60979f-d710-44f9-a406-b19280787ed3 tempest-FloatingIPsAssociationNegativeTestJSON-1553180457 tempest-FloatingIPsAssociationNegativeTestJSON-1553180457-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:07:52.693799 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-0f60979f-d710-44f9-a406-b19280787ed3 tempest-FloatingIPsAssociationNegativeTestJSON-1553180457 tempest-FloatingIPsAssociationNegativeTestJSON-1553180457-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.003s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:07:52.694955 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-0f60979f-d710-44f9-a406-b19280787ed3 tempest-FloatingIPsAssociationNegativeTestJSON-1553180457 tempest-FloatingIPsAssociationNegativeTestJSON-1553180457-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.003s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:07:52.772816 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-0f60979f-d710-44f9-a406-b19280787ed3 tempest-FloatingIPsAssociationNegativeTestJSON-1553180457 tempest-FloatingIPsAssociationNegativeTestJSON-1553180457-project] [instance: f4ad0f65-5439-4fd9-9d2d-5abda464364a] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='87b0fec9-a948-41f9-a3c0-f0a6a5a8f1e7',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120658) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 18 22:07:52.774855 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-0f60979f-d710-44f9-a406-b19280787ed3 tempest-FloatingIPsAssociationNegativeTestJSON-1553180457 tempest-FloatingIPsAssociationNegativeTestJSON-1553180457-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:07:52.775719 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-0f60979f-d710-44f9-a406-b19280787ed3 tempest-FloatingIPsAssociationNegativeTestJSON-1553180457 tempest-FloatingIPsAssociationNegativeTestJSON-1553180457-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:07:52.776348 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-0f60979f-d710-44f9-a406-b19280787ed3 tempest-FloatingIPsAssociationNegativeTestJSON-1553180457 tempest-FloatingIPsAssociationNegativeTestJSON-1553180457-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:07:53.084098 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-0f60979f-d710-44f9-a406-b19280787ed3 tempest-FloatingIPsAssociationNegativeTestJSON-1553180457 tempest-FloatingIPsAssociationNegativeTestJSON-1553180457-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:07:53.098434 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-0f60979f-d710-44f9-a406-b19280787ed3 tempest-FloatingIPsAssociationNegativeTestJSON-1553180457 tempest-FloatingIPsAssociationNegativeTestJSON-1553180457-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:07:53.099613 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-0f60979f-d710-44f9-a406-b19280787ed3 tempest-FloatingIPsAssociationNegativeTestJSON-1553180457 tempest-FloatingIPsAssociationNegativeTestJSON-1553180457-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.014s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:07:53.102183 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-0f60979f-d710-44f9-a406-b19280787ed3 tempest-FloatingIPsAssociationNegativeTestJSON-1553180457 tempest-FloatingIPsAssociationNegativeTestJSON-1553180457-project] Acquiring lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 18 22:07:53.103006 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-0f60979f-d710-44f9-a406-b19280787ed3 tempest-FloatingIPsAssociationNegativeTestJSON-1553180457 tempest-FloatingIPsAssociationNegativeTestJSON-1553180457-project] Acquired lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 18 22:07:53.123756 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-a6c4d5d9-3ef1-4c3b-97df-4c005fb27fd3 None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:07:53.127465 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-a6c4d5d9-3ef1-4c3b-97df-4c005fb27fd3 None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.020s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:07:53.128429 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-a6c4d5d9-3ef1-4c3b-97df-4c005fb27fd3 None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.004s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:07:53.132613 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-a6c4d5d9-3ef1-4c3b-97df-4c005fb27fd3 None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:07:53.133600 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-a6c4d5d9-3ef1-4c3b-97df-4c005fb27fd3 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.004s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:07:53.134877 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-a6c4d5d9-3ef1-4c3b-97df-4c005fb27fd3 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:07:53.173995 n-d-853704-1 nova-conductor[120658]: ERROR nova.compute.rpcapi [None req-a6c4d5d9-3ef1-4c3b-97df-4c005fb27fd3 None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 18 22:07:53.174819 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-a6c4d5d9-3ef1-4c3b-97df-4c005fb27fd3 None None] Releasing lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 18 22:07:53.176738 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server [None req-a6c4d5d9-3ef1-4c3b-97df-4c005fb27fd3 None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:07:53.176738 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:07:53.176738 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 18 22:07:53.176738 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 18 22:07:53.176738 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 18 22:07:53.176738 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 22:07:53.176738 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 18 22:07:53.176738 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 22:07:53.176738 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:07:53.176738 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 18 22:07:53.176738 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 18 22:07:53.176738 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 18 22:07:53.176738 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 18 22:07:53.176738 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 18 22:07:53.176738 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 18 22:07:53.176738 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 18 22:07:53.176738 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 18 22:07:53.179969 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 18 22:07:53.179969 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 18 22:07:53.179969 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 18 22:07:53.179969 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 18 22:07:53.179969 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 18 22:07:53.179969 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 18 22:07:53.179969 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:07:53.179969 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 22:07:54.596063 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-10be983e-5cb6-4a46-a790-25cde10060ee tempest-LiveMigrationTest-1883958412 tempest-LiveMigrationTest-1883958412-project] Took 0.67 seconds to select destinations for 1 instance(s). {{(pid=120657) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 18 22:07:54.634736 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-10be983e-5cb6-4a46-a790-25cde10060ee tempest-LiveMigrationTest-1883958412 tempest-LiveMigrationTest-1883958412-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:07:54.635310 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-10be983e-5cb6-4a46-a790-25cde10060ee tempest-LiveMigrationTest-1883958412 tempest-LiveMigrationTest-1883958412-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:07:54.635738 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-10be983e-5cb6-4a46-a790-25cde10060ee tempest-LiveMigrationTest-1883958412 tempest-LiveMigrationTest-1883958412-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:07:54.846154 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-10be983e-5cb6-4a46-a790-25cde10060ee tempest-LiveMigrationTest-1883958412 tempest-LiveMigrationTest-1883958412-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:07:54.846959 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-10be983e-5cb6-4a46-a790-25cde10060ee tempest-LiveMigrationTest-1883958412 tempest-LiveMigrationTest-1883958412-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:07:54.848098 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-10be983e-5cb6-4a46-a790-25cde10060ee tempest-LiveMigrationTest-1883958412 tempest-LiveMigrationTest-1883958412-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:07:54.849186 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-10be983e-5cb6-4a46-a790-25cde10060ee tempest-LiveMigrationTest-1883958412 tempest-LiveMigrationTest-1883958412-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:07:54.849932 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-10be983e-5cb6-4a46-a790-25cde10060ee tempest-LiveMigrationTest-1883958412 tempest-LiveMigrationTest-1883958412-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:07:54.850663 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-10be983e-5cb6-4a46-a790-25cde10060ee tempest-LiveMigrationTest-1883958412 tempest-LiveMigrationTest-1883958412-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:07:54.876933 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-10be983e-5cb6-4a46-a790-25cde10060ee tempest-LiveMigrationTest-1883958412 tempest-LiveMigrationTest-1883958412-project] Getting quotas for project 29de488917ec43a09800ec6cb2187b56. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 18 22:07:54.882658 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-10be983e-5cb6-4a46-a790-25cde10060ee tempest-LiveMigrationTest-1883958412 tempest-LiveMigrationTest-1883958412-project] Getting quotas for user c137b718cd2445909cacb17658bff8f6 and project 29de488917ec43a09800ec6cb2187b56. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 18 22:07:54.901661 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-10be983e-5cb6-4a46-a790-25cde10060ee tempest-LiveMigrationTest-1883958412 tempest-LiveMigrationTest-1883958412-project] [instance: 282be306-3b11-4ff7-ade3-df294fb54375] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120657) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 18 22:07:54.904379 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-10be983e-5cb6-4a46-a790-25cde10060ee tempest-LiveMigrationTest-1883958412 tempest-LiveMigrationTest-1883958412-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:07:54.904828 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-10be983e-5cb6-4a46-a790-25cde10060ee tempest-LiveMigrationTest-1883958412 tempest-LiveMigrationTest-1883958412-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:07:54.905226 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-10be983e-5cb6-4a46-a790-25cde10060ee tempest-LiveMigrationTest-1883958412 tempest-LiveMigrationTest-1883958412-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:07:54.975185 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-10be983e-5cb6-4a46-a790-25cde10060ee tempest-LiveMigrationTest-1883958412 tempest-LiveMigrationTest-1883958412-project] [instance: 282be306-3b11-4ff7-ade3-df294fb54375] block_device_mapping [BlockDeviceMapping(attachment_id=20a7d262-8749-4bfe-ad91-9a8b6c04aa0d,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='volume',device_name=None,device_type=None,disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id=None,instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='volume',tag=None,updated_at=,uuid=,volume_id='b111bef3-0cdb-4dfe-a605-4f8b959d0758',volume_size=1,volume_type=None)] {{(pid=120657) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 18 22:07:54.976770 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-10be983e-5cb6-4a46-a790-25cde10060ee tempest-LiveMigrationTest-1883958412 tempest-LiveMigrationTest-1883958412-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:07:54.977368 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-10be983e-5cb6-4a46-a790-25cde10060ee tempest-LiveMigrationTest-1883958412 tempest-LiveMigrationTest-1883958412-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:07:54.977924 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-10be983e-5cb6-4a46-a790-25cde10060ee tempest-LiveMigrationTest-1883958412 tempest-LiveMigrationTest-1883958412-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:07:55.205347 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-10be983e-5cb6-4a46-a790-25cde10060ee tempest-LiveMigrationTest-1883958412 tempest-LiveMigrationTest-1883958412-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:07:55.205804 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-10be983e-5cb6-4a46-a790-25cde10060ee tempest-LiveMigrationTest-1883958412 tempest-LiveMigrationTest-1883958412-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:07:55.206272 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-10be983e-5cb6-4a46-a790-25cde10060ee tempest-LiveMigrationTest-1883958412 tempest-LiveMigrationTest-1883958412-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:07:55.206732 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-10be983e-5cb6-4a46-a790-25cde10060ee tempest-LiveMigrationTest-1883958412 tempest-LiveMigrationTest-1883958412-project] Acquiring lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 18 22:07:55.207300 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-10be983e-5cb6-4a46-a790-25cde10060ee tempest-LiveMigrationTest-1883958412 tempest-LiveMigrationTest-1883958412-project] Acquired lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 18 22:07:55.211205 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-dc41b694-718c-49ea-b83f-ee9e1f2856c5 None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:07:55.221820 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-dc41b694-718c-49ea-b83f-ee9e1f2856c5 None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.004s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:07:55.221820 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-dc41b694-718c-49ea-b83f-ee9e1f2856c5 None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:07:55.221820 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-dc41b694-718c-49ea-b83f-ee9e1f2856c5 None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:07:55.221820 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-dc41b694-718c-49ea-b83f-ee9e1f2856c5 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:07:55.221820 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-dc41b694-718c-49ea-b83f-ee9e1f2856c5 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:07:55.233936 n-d-853704-1 nova-conductor[120657]: ERROR nova.compute.rpcapi [None req-dc41b694-718c-49ea-b83f-ee9e1f2856c5 None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 18 22:07:55.234339 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-dc41b694-718c-49ea-b83f-ee9e1f2856c5 None None] Releasing lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 18 22:07:55.235270 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server [None req-dc41b694-718c-49ea-b83f-ee9e1f2856c5 None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:07:55.235270 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:07:55.235270 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 18 22:07:55.235270 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 18 22:07:55.235270 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 18 22:07:55.235270 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 22:07:55.235270 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 18 22:07:55.235270 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 22:07:55.235270 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:07:55.235270 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 18 22:07:55.235270 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 18 22:07:55.235270 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 18 22:07:55.235270 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 18 22:07:55.235270 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 18 22:07:55.235270 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 18 22:07:55.235270 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 18 22:07:55.235270 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 18 22:07:55.244574 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 18 22:07:55.244574 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 18 22:07:55.244574 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 18 22:07:55.244574 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 18 22:07:55.244574 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 18 22:07:55.244574 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 18 22:07:55.244574 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:07:55.244574 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 22:08:02.893140 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=11,nova_api:UPDATE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:08:02.978372 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=13,nova_api:DELETE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:08:03.173789 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:08:03.173789 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell1:SELECT=11,nova_cell1:SAVEPOINT=1,nova_cell1:INSERT=18,nova_cell1:RELEASE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:08:05.132203 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=10 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:08:05.134439 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=14,nova_api:UPDATE=1,nova_api:DELETE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:08:05.234323 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:08:05.235361 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell1:SELECT=11,nova_cell1:INSERT=16 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:08:12.949660 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=12,nova_cell0:UPDATE=3 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:08:12.987854 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=2 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:08:23.144473 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-4636432a-7631-4d74-8802-fa4ab14bc217 tempest-ListServersNegativeTestJSON-1327008128 tempest-ListServersNegativeTestJSON-1327008128-project] Took 1.70 seconds to select destinations for 3 instance(s). {{(pid=120658) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 18 22:08:23.168772 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-4636432a-7631-4d74-8802-fa4ab14bc217 tempest-ListServersNegativeTestJSON-1327008128 tempest-ListServersNegativeTestJSON-1327008128-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:08:23.169222 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-4636432a-7631-4d74-8802-fa4ab14bc217 tempest-ListServersNegativeTestJSON-1327008128 tempest-ListServersNegativeTestJSON-1327008128-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:08:23.169619 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-4636432a-7631-4d74-8802-fa4ab14bc217 tempest-ListServersNegativeTestJSON-1327008128 tempest-ListServersNegativeTestJSON-1327008128-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:08:23.506842 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-4636432a-7631-4d74-8802-fa4ab14bc217 tempest-ListServersNegativeTestJSON-1327008128 tempest-ListServersNegativeTestJSON-1327008128-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:08:23.507057 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-4636432a-7631-4d74-8802-fa4ab14bc217 tempest-ListServersNegativeTestJSON-1327008128 tempest-ListServersNegativeTestJSON-1327008128-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:08:23.507442 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-4636432a-7631-4d74-8802-fa4ab14bc217 tempest-ListServersNegativeTestJSON-1327008128 tempest-ListServersNegativeTestJSON-1327008128-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:08:23.730586 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-4636432a-7631-4d74-8802-fa4ab14bc217 tempest-ListServersNegativeTestJSON-1327008128 tempest-ListServersNegativeTestJSON-1327008128-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:08:23.731034 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-4636432a-7631-4d74-8802-fa4ab14bc217 tempest-ListServersNegativeTestJSON-1327008128 tempest-ListServersNegativeTestJSON-1327008128-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:08:23.731437 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-4636432a-7631-4d74-8802-fa4ab14bc217 tempest-ListServersNegativeTestJSON-1327008128 tempest-ListServersNegativeTestJSON-1327008128-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:08:23.854466 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-4636432a-7631-4d74-8802-fa4ab14bc217 tempest-ListServersNegativeTestJSON-1327008128 tempest-ListServersNegativeTestJSON-1327008128-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:08:23.854890 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-4636432a-7631-4d74-8802-fa4ab14bc217 tempest-ListServersNegativeTestJSON-1327008128 tempest-ListServersNegativeTestJSON-1327008128-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:08:23.855278 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-4636432a-7631-4d74-8802-fa4ab14bc217 tempest-ListServersNegativeTestJSON-1327008128 tempest-ListServersNegativeTestJSON-1327008128-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:08:23.856038 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-4636432a-7631-4d74-8802-fa4ab14bc217 tempest-ListServersNegativeTestJSON-1327008128 tempest-ListServersNegativeTestJSON-1327008128-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:08:23.856451 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-4636432a-7631-4d74-8802-fa4ab14bc217 tempest-ListServersNegativeTestJSON-1327008128 tempest-ListServersNegativeTestJSON-1327008128-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:08:23.856833 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-4636432a-7631-4d74-8802-fa4ab14bc217 tempest-ListServersNegativeTestJSON-1327008128 tempest-ListServersNegativeTestJSON-1327008128-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:08:23.872796 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-4636432a-7631-4d74-8802-fa4ab14bc217 tempest-ListServersNegativeTestJSON-1327008128 tempest-ListServersNegativeTestJSON-1327008128-project] Getting quotas for project 3efa3556ee504e6b8c67e2e3a4e76831. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 18 22:08:23.877892 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-4636432a-7631-4d74-8802-fa4ab14bc217 tempest-ListServersNegativeTestJSON-1327008128 tempest-ListServersNegativeTestJSON-1327008128-project] Getting quotas for user 37e4bacf0c0b43b181760d2f3591e49b and project 3efa3556ee504e6b8c67e2e3a4e76831. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 18 22:08:23.889620 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-4636432a-7631-4d74-8802-fa4ab14bc217 tempest-ListServersNegativeTestJSON-1327008128 tempest-ListServersNegativeTestJSON-1327008128-project] [instance: 2ec86486-0b77-41fc-9c73-27989b9f5ed4] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [] {{(pid=120658) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 18 22:08:23.890458 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-4636432a-7631-4d74-8802-fa4ab14bc217 tempest-ListServersNegativeTestJSON-1327008128 tempest-ListServersNegativeTestJSON-1327008128-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:08:23.890940 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-4636432a-7631-4d74-8802-fa4ab14bc217 tempest-ListServersNegativeTestJSON-1327008128 tempest-ListServersNegativeTestJSON-1327008128-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:08:23.891333 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-4636432a-7631-4d74-8802-fa4ab14bc217 tempest-ListServersNegativeTestJSON-1327008128 tempest-ListServersNegativeTestJSON-1327008128-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:08:23.977213 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-4636432a-7631-4d74-8802-fa4ab14bc217 tempest-ListServersNegativeTestJSON-1327008128 tempest-ListServersNegativeTestJSON-1327008128-project] [instance: 2ec86486-0b77-41fc-9c73-27989b9f5ed4] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='87b0fec9-a948-41f9-a3c0-f0a6a5a8f1e7',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120658) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 18 22:08:23.978534 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-4636432a-7631-4d74-8802-fa4ab14bc217 tempest-ListServersNegativeTestJSON-1327008128 tempest-ListServersNegativeTestJSON-1327008128-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:08:23.979177 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-4636432a-7631-4d74-8802-fa4ab14bc217 tempest-ListServersNegativeTestJSON-1327008128 tempest-ListServersNegativeTestJSON-1327008128-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:08:23.979657 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-4636432a-7631-4d74-8802-fa4ab14bc217 tempest-ListServersNegativeTestJSON-1327008128 tempest-ListServersNegativeTestJSON-1327008128-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:08:24.338309 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-4636432a-7631-4d74-8802-fa4ab14bc217 tempest-ListServersNegativeTestJSON-1327008128 tempest-ListServersNegativeTestJSON-1327008128-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:08:24.338309 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-4636432a-7631-4d74-8802-fa4ab14bc217 tempest-ListServersNegativeTestJSON-1327008128 tempest-ListServersNegativeTestJSON-1327008128-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:08:24.338309 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-4636432a-7631-4d74-8802-fa4ab14bc217 tempest-ListServersNegativeTestJSON-1327008128 tempest-ListServersNegativeTestJSON-1327008128-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:08:24.339558 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-4636432a-7631-4d74-8802-fa4ab14bc217 tempest-ListServersNegativeTestJSON-1327008128 tempest-ListServersNegativeTestJSON-1327008128-project] Acquiring lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 18 22:08:24.339558 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-4636432a-7631-4d74-8802-fa4ab14bc217 tempest-ListServersNegativeTestJSON-1327008128 tempest-ListServersNegativeTestJSON-1327008128-project] Acquired lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 18 22:08:24.344017 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-cb710f25-53cd-455e-a201-988541c52a40 None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:08:24.344516 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-cb710f25-53cd-455e-a201-988541c52a40 None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.005s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:08:24.344925 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-cb710f25-53cd-455e-a201-988541c52a40 None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:08:24.349732 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-cb710f25-53cd-455e-a201-988541c52a40 None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:08:24.349732 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-cb710f25-53cd-455e-a201-988541c52a40 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:08:24.349732 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-cb710f25-53cd-455e-a201-988541c52a40 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:08:24.364878 n-d-853704-1 nova-conductor[120658]: ERROR nova.compute.rpcapi [None req-cb710f25-53cd-455e-a201-988541c52a40 None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 18 22:08:24.368784 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-cb710f25-53cd-455e-a201-988541c52a40 None None] Releasing lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 18 22:08:24.375605 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server [None req-cb710f25-53cd-455e-a201-988541c52a40 None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:08:24.375605 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:08:24.375605 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 18 22:08:24.375605 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 18 22:08:24.375605 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 18 22:08:24.375605 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 22:08:24.375605 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 18 22:08:24.375605 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 22:08:24.375605 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:08:24.375605 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 18 22:08:24.375605 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 18 22:08:24.375605 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 18 22:08:24.375605 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 18 22:08:24.375605 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 18 22:08:24.375605 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 18 22:08:24.375605 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 18 22:08:24.375605 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 18 22:08:24.377093 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 18 22:08:24.377093 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 18 22:08:24.377093 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 18 22:08:24.377093 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 18 22:08:24.377093 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 18 22:08:24.377093 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 18 22:08:24.377093 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:08:24.377093 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 22:08:32.238448 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-ac873db2-dcd4-4926-b3a2-37467ac579c8 tempest-LiveAutoBlockMigrationV225Test-1869140755 tempest-LiveAutoBlockMigrationV225Test-1869140755-project] Took 0.41 seconds to select destinations for 1 instance(s). {{(pid=120657) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 18 22:08:32.259941 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-ac873db2-dcd4-4926-b3a2-37467ac579c8 tempest-LiveAutoBlockMigrationV225Test-1869140755 tempest-LiveAutoBlockMigrationV225Test-1869140755-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:08:32.260395 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-ac873db2-dcd4-4926-b3a2-37467ac579c8 tempest-LiveAutoBlockMigrationV225Test-1869140755 tempest-LiveAutoBlockMigrationV225Test-1869140755-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:08:32.260797 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-ac873db2-dcd4-4926-b3a2-37467ac579c8 tempest-LiveAutoBlockMigrationV225Test-1869140755 tempest-LiveAutoBlockMigrationV225Test-1869140755-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:08:32.397345 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-ac873db2-dcd4-4926-b3a2-37467ac579c8 tempest-LiveAutoBlockMigrationV225Test-1869140755 tempest-LiveAutoBlockMigrationV225Test-1869140755-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:08:32.397793 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-ac873db2-dcd4-4926-b3a2-37467ac579c8 tempest-LiveAutoBlockMigrationV225Test-1869140755 tempest-LiveAutoBlockMigrationV225Test-1869140755-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:08:32.398191 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-ac873db2-dcd4-4926-b3a2-37467ac579c8 tempest-LiveAutoBlockMigrationV225Test-1869140755 tempest-LiveAutoBlockMigrationV225Test-1869140755-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:08:32.398858 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-ac873db2-dcd4-4926-b3a2-37467ac579c8 tempest-LiveAutoBlockMigrationV225Test-1869140755 tempest-LiveAutoBlockMigrationV225Test-1869140755-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:08:32.399281 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-ac873db2-dcd4-4926-b3a2-37467ac579c8 tempest-LiveAutoBlockMigrationV225Test-1869140755 tempest-LiveAutoBlockMigrationV225Test-1869140755-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:08:32.399686 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-ac873db2-dcd4-4926-b3a2-37467ac579c8 tempest-LiveAutoBlockMigrationV225Test-1869140755 tempest-LiveAutoBlockMigrationV225Test-1869140755-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:08:32.416075 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-ac873db2-dcd4-4926-b3a2-37467ac579c8 tempest-LiveAutoBlockMigrationV225Test-1869140755 tempest-LiveAutoBlockMigrationV225Test-1869140755-project] Getting quotas for project 4c471d428ca241faa5d631d226526b3b. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 18 22:08:32.420167 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-ac873db2-dcd4-4926-b3a2-37467ac579c8 tempest-LiveAutoBlockMigrationV225Test-1869140755 tempest-LiveAutoBlockMigrationV225Test-1869140755-project] Getting quotas for user faee7152cc454a189fcbdd8f2d4b7b29 and project 4c471d428ca241faa5d631d226526b3b. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 18 22:08:32.431705 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-ac873db2-dcd4-4926-b3a2-37467ac579c8 tempest-LiveAutoBlockMigrationV225Test-1869140755 tempest-LiveAutoBlockMigrationV225Test-1869140755-project] [instance: e711f789-9be7-4885-acd9-6b8a6d81f1c6] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120657) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 18 22:08:32.432488 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-ac873db2-dcd4-4926-b3a2-37467ac579c8 tempest-LiveAutoBlockMigrationV225Test-1869140755 tempest-LiveAutoBlockMigrationV225Test-1869140755-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:08:32.432931 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-ac873db2-dcd4-4926-b3a2-37467ac579c8 tempest-LiveAutoBlockMigrationV225Test-1869140755 tempest-LiveAutoBlockMigrationV225Test-1869140755-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:08:32.433329 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-ac873db2-dcd4-4926-b3a2-37467ac579c8 tempest-LiveAutoBlockMigrationV225Test-1869140755 tempest-LiveAutoBlockMigrationV225Test-1869140755-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:08:32.489440 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-ac873db2-dcd4-4926-b3a2-37467ac579c8 tempest-LiveAutoBlockMigrationV225Test-1869140755 tempest-LiveAutoBlockMigrationV225Test-1869140755-project] [instance: e711f789-9be7-4885-acd9-6b8a6d81f1c6] block_device_mapping [BlockDeviceMapping(attachment_id=c3c2b009-b4ad-4e5c-b409-81f8e4b994fc,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='volume',device_name=None,device_type=None,disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id=None,instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='volume',tag=None,updated_at=,uuid=,volume_id='d41780e1-3dc3-4f9f-aaf2-4c446f330256',volume_size=1,volume_type=None)] {{(pid=120657) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 18 22:08:32.491371 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-ac873db2-dcd4-4926-b3a2-37467ac579c8 tempest-LiveAutoBlockMigrationV225Test-1869140755 tempest-LiveAutoBlockMigrationV225Test-1869140755-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:08:32.491954 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-ac873db2-dcd4-4926-b3a2-37467ac579c8 tempest-LiveAutoBlockMigrationV225Test-1869140755 tempest-LiveAutoBlockMigrationV225Test-1869140755-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:08:32.492362 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-ac873db2-dcd4-4926-b3a2-37467ac579c8 tempest-LiveAutoBlockMigrationV225Test-1869140755 tempest-LiveAutoBlockMigrationV225Test-1869140755-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:08:32.731396 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-ac873db2-dcd4-4926-b3a2-37467ac579c8 tempest-LiveAutoBlockMigrationV225Test-1869140755 tempest-LiveAutoBlockMigrationV225Test-1869140755-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:08:32.731396 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-ac873db2-dcd4-4926-b3a2-37467ac579c8 tempest-LiveAutoBlockMigrationV225Test-1869140755 tempest-LiveAutoBlockMigrationV225Test-1869140755-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:08:32.731614 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-ac873db2-dcd4-4926-b3a2-37467ac579c8 tempest-LiveAutoBlockMigrationV225Test-1869140755 tempest-LiveAutoBlockMigrationV225Test-1869140755-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:08:32.731975 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-ac873db2-dcd4-4926-b3a2-37467ac579c8 tempest-LiveAutoBlockMigrationV225Test-1869140755 tempest-LiveAutoBlockMigrationV225Test-1869140755-project] Acquiring lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 18 22:08:32.732355 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-ac873db2-dcd4-4926-b3a2-37467ac579c8 tempest-LiveAutoBlockMigrationV225Test-1869140755 tempest-LiveAutoBlockMigrationV225Test-1869140755-project] Acquired lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 18 22:08:32.733115 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-9c979075-497e-4b40-989e-9ab870d96c43 None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:08:32.733527 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-9c979075-497e-4b40-989e-9ab870d96c43 None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:08:32.733908 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-9c979075-497e-4b40-989e-9ab870d96c43 None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:08:32.734546 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-9c979075-497e-4b40-989e-9ab870d96c43 None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:08:32.734953 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-9c979075-497e-4b40-989e-9ab870d96c43 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:08:32.735329 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-9c979075-497e-4b40-989e-9ab870d96c43 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:08:32.746368 n-d-853704-1 nova-conductor[120657]: ERROR nova.compute.rpcapi [None req-9c979075-497e-4b40-989e-9ab870d96c43 None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 18 22:08:32.746759 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-9c979075-497e-4b40-989e-9ab870d96c43 None None] Releasing lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 18 22:08:32.747657 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server [None req-9c979075-497e-4b40-989e-9ab870d96c43 None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:08:32.747657 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:08:32.747657 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 18 22:08:32.747657 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 18 22:08:32.747657 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 18 22:08:32.747657 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 22:08:32.747657 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 18 22:08:32.747657 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 22:08:32.747657 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:08:32.747657 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 18 22:08:32.747657 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 18 22:08:32.747657 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 18 22:08:32.747657 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 18 22:08:32.747657 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 18 22:08:32.747657 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 18 22:08:32.747657 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 18 22:08:32.747657 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 18 22:08:32.748958 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 18 22:08:32.748958 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 18 22:08:32.748958 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 18 22:08:32.748958 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 18 22:08:32.748958 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 18 22:08:32.748958 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 18 22:08:32.748958 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:08:32.748958 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 22:08:34.217090 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=20 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:08:34.218218 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=12,nova_api:UPDATE=1,nova_api:DELETE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:08:34.365084 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:08:34.365747 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell1:SELECT=21,nova_cell1:SAVEPOINT=1,nova_cell1:INSERT=48,nova_cell1:RELEASE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:08:42.592213 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=11,nova_api:UPDATE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:08:42.648648 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=13,nova_api:DELETE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:08:42.743187 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:08:42.743929 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell1:SELECT=11,nova_cell1:INSERT=16 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:08:42.852051 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-18f1d28b-1911-42e2-8b94-7ad260d5a158 tempest-ImagesOneServerNegativeTestJSON-184634117 tempest-ImagesOneServerNegativeTestJSON-184634117-project] Took 0.66 seconds to select destinations for 1 instance(s). {{(pid=120658) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 18 22:08:42.877814 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-18f1d28b-1911-42e2-8b94-7ad260d5a158 tempest-ImagesOneServerNegativeTestJSON-184634117 tempest-ImagesOneServerNegativeTestJSON-184634117-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:08:42.878830 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-18f1d28b-1911-42e2-8b94-7ad260d5a158 tempest-ImagesOneServerNegativeTestJSON-184634117 tempest-ImagesOneServerNegativeTestJSON-184634117-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:08:42.879394 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-18f1d28b-1911-42e2-8b94-7ad260d5a158 tempest-ImagesOneServerNegativeTestJSON-184634117 tempest-ImagesOneServerNegativeTestJSON-184634117-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:08:43.233230 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-18f1d28b-1911-42e2-8b94-7ad260d5a158 tempest-ImagesOneServerNegativeTestJSON-184634117 tempest-ImagesOneServerNegativeTestJSON-184634117-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:08:43.233675 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-18f1d28b-1911-42e2-8b94-7ad260d5a158 tempest-ImagesOneServerNegativeTestJSON-184634117 tempest-ImagesOneServerNegativeTestJSON-184634117-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:08:43.234584 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-18f1d28b-1911-42e2-8b94-7ad260d5a158 tempest-ImagesOneServerNegativeTestJSON-184634117 tempest-ImagesOneServerNegativeTestJSON-184634117-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:08:43.234750 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-18f1d28b-1911-42e2-8b94-7ad260d5a158 tempest-ImagesOneServerNegativeTestJSON-184634117 tempest-ImagesOneServerNegativeTestJSON-184634117-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:08:43.235163 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-18f1d28b-1911-42e2-8b94-7ad260d5a158 tempest-ImagesOneServerNegativeTestJSON-184634117 tempest-ImagesOneServerNegativeTestJSON-184634117-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:08:43.235548 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-18f1d28b-1911-42e2-8b94-7ad260d5a158 tempest-ImagesOneServerNegativeTestJSON-184634117 tempest-ImagesOneServerNegativeTestJSON-184634117-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:08:43.251840 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-18f1d28b-1911-42e2-8b94-7ad260d5a158 tempest-ImagesOneServerNegativeTestJSON-184634117 tempest-ImagesOneServerNegativeTestJSON-184634117-project] Getting quotas for project 95af50ab8c974f07b41711d288115816. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 18 22:08:43.256385 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-18f1d28b-1911-42e2-8b94-7ad260d5a158 tempest-ImagesOneServerNegativeTestJSON-184634117 tempest-ImagesOneServerNegativeTestJSON-184634117-project] Getting quotas for user d2f9029137e0471384c95625833273c7 and project 95af50ab8c974f07b41711d288115816. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 18 22:08:43.269131 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-18f1d28b-1911-42e2-8b94-7ad260d5a158 tempest-ImagesOneServerNegativeTestJSON-184634117 tempest-ImagesOneServerNegativeTestJSON-184634117-project] [instance: 3b8e2f6e-c4d2-4e6d-87bf-920beae9f379] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120658) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 18 22:08:43.270176 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-18f1d28b-1911-42e2-8b94-7ad260d5a158 tempest-ImagesOneServerNegativeTestJSON-184634117 tempest-ImagesOneServerNegativeTestJSON-184634117-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:08:43.270621 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-18f1d28b-1911-42e2-8b94-7ad260d5a158 tempest-ImagesOneServerNegativeTestJSON-184634117 tempest-ImagesOneServerNegativeTestJSON-184634117-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:08:43.271021 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-18f1d28b-1911-42e2-8b94-7ad260d5a158 tempest-ImagesOneServerNegativeTestJSON-184634117 tempest-ImagesOneServerNegativeTestJSON-184634117-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:08:43.332818 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-18f1d28b-1911-42e2-8b94-7ad260d5a158 tempest-ImagesOneServerNegativeTestJSON-184634117 tempest-ImagesOneServerNegativeTestJSON-184634117-project] [instance: 3b8e2f6e-c4d2-4e6d-87bf-920beae9f379] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='87b0fec9-a948-41f9-a3c0-f0a6a5a8f1e7',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120658) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 18 22:08:43.334700 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-18f1d28b-1911-42e2-8b94-7ad260d5a158 tempest-ImagesOneServerNegativeTestJSON-184634117 tempest-ImagesOneServerNegativeTestJSON-184634117-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:08:43.335785 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-18f1d28b-1911-42e2-8b94-7ad260d5a158 tempest-ImagesOneServerNegativeTestJSON-184634117 tempest-ImagesOneServerNegativeTestJSON-184634117-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:08:43.336195 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-18f1d28b-1911-42e2-8b94-7ad260d5a158 tempest-ImagesOneServerNegativeTestJSON-184634117 tempest-ImagesOneServerNegativeTestJSON-184634117-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:08:43.639007 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-18f1d28b-1911-42e2-8b94-7ad260d5a158 tempest-ImagesOneServerNegativeTestJSON-184634117 tempest-ImagesOneServerNegativeTestJSON-184634117-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:08:43.639549 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-18f1d28b-1911-42e2-8b94-7ad260d5a158 tempest-ImagesOneServerNegativeTestJSON-184634117 tempest-ImagesOneServerNegativeTestJSON-184634117-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:08:43.640078 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-18f1d28b-1911-42e2-8b94-7ad260d5a158 tempest-ImagesOneServerNegativeTestJSON-184634117 tempest-ImagesOneServerNegativeTestJSON-184634117-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:08:43.640620 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-18f1d28b-1911-42e2-8b94-7ad260d5a158 tempest-ImagesOneServerNegativeTestJSON-184634117 tempest-ImagesOneServerNegativeTestJSON-184634117-project] Acquiring lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 18 22:08:43.641051 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-18f1d28b-1911-42e2-8b94-7ad260d5a158 tempest-ImagesOneServerNegativeTestJSON-184634117 tempest-ImagesOneServerNegativeTestJSON-184634117-project] Acquired lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 18 22:08:43.641846 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-cc31d72d-07c0-40ed-8d6f-95fa2366f30c None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:08:43.642324 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-cc31d72d-07c0-40ed-8d6f-95fa2366f30c None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:08:43.642774 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-cc31d72d-07c0-40ed-8d6f-95fa2366f30c None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:08:43.643518 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-cc31d72d-07c0-40ed-8d6f-95fa2366f30c None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:08:43.644017 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-cc31d72d-07c0-40ed-8d6f-95fa2366f30c None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:08:43.644455 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-cc31d72d-07c0-40ed-8d6f-95fa2366f30c None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:08:43.662409 n-d-853704-1 nova-conductor[120658]: ERROR nova.compute.rpcapi [None req-cc31d72d-07c0-40ed-8d6f-95fa2366f30c None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 18 22:08:43.662814 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-cc31d72d-07c0-40ed-8d6f-95fa2366f30c None None] Releasing lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 18 22:08:43.663778 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server [None req-cc31d72d-07c0-40ed-8d6f-95fa2366f30c None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:08:43.663778 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:08:43.663778 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 18 22:08:43.663778 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 18 22:08:43.663778 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 18 22:08:43.663778 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 22:08:43.663778 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 18 22:08:43.663778 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 22:08:43.663778 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:08:43.663778 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 18 22:08:43.663778 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 18 22:08:43.663778 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 18 22:08:43.663778 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 18 22:08:43.663778 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 18 22:08:43.663778 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 18 22:08:43.663778 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 18 22:08:43.663778 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 18 22:08:43.665010 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 18 22:08:43.665010 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 18 22:08:43.665010 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 18 22:08:43.665010 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 18 22:08:43.665010 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 18 22:08:43.665010 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 18 22:08:43.665010 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:08:43.665010 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 22:08:44.023755 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-1e6cebf0-8316-49e5-9220-a56a7e402dbc tempest-ServerDiskConfigTestJSON-1657751511 tempest-ServerDiskConfigTestJSON-1657751511-project] Took 0.51 seconds to select destinations for 1 instance(s). {{(pid=120657) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 18 22:08:44.056054 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-1e6cebf0-8316-49e5-9220-a56a7e402dbc tempest-ServerDiskConfigTestJSON-1657751511 tempest-ServerDiskConfigTestJSON-1657751511-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:08:44.056870 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-1e6cebf0-8316-49e5-9220-a56a7e402dbc tempest-ServerDiskConfigTestJSON-1657751511 tempest-ServerDiskConfigTestJSON-1657751511-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.005s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:08:44.057143 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-1e6cebf0-8316-49e5-9220-a56a7e402dbc tempest-ServerDiskConfigTestJSON-1657751511 tempest-ServerDiskConfigTestJSON-1657751511-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:08:44.194117 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=15,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:08:44.197523 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-1e6cebf0-8316-49e5-9220-a56a7e402dbc tempest-ServerDiskConfigTestJSON-1657751511 tempest-ServerDiskConfigTestJSON-1657751511-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:08:44.198094 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-1e6cebf0-8316-49e5-9220-a56a7e402dbc tempest-ServerDiskConfigTestJSON-1657751511 tempest-ServerDiskConfigTestJSON-1657751511-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:08:44.198794 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-1e6cebf0-8316-49e5-9220-a56a7e402dbc tempest-ServerDiskConfigTestJSON-1657751511 tempest-ServerDiskConfigTestJSON-1657751511-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:08:44.200167 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-1e6cebf0-8316-49e5-9220-a56a7e402dbc tempest-ServerDiskConfigTestJSON-1657751511 tempest-ServerDiskConfigTestJSON-1657751511-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:08:44.200630 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-1e6cebf0-8316-49e5-9220-a56a7e402dbc tempest-ServerDiskConfigTestJSON-1657751511 tempest-ServerDiskConfigTestJSON-1657751511-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:08:44.201029 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-1e6cebf0-8316-49e5-9220-a56a7e402dbc tempest-ServerDiskConfigTestJSON-1657751511 tempest-ServerDiskConfigTestJSON-1657751511-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:08:44.219465 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-1e6cebf0-8316-49e5-9220-a56a7e402dbc tempest-ServerDiskConfigTestJSON-1657751511 tempest-ServerDiskConfigTestJSON-1657751511-project] Getting quotas for project 4450d6767020433e986d18bb2965de51. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 18 22:08:44.224553 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-1e6cebf0-8316-49e5-9220-a56a7e402dbc tempest-ServerDiskConfigTestJSON-1657751511 tempest-ServerDiskConfigTestJSON-1657751511-project] Getting quotas for user 9a6f17cb5745439882594505d7f20df1 and project 4450d6767020433e986d18bb2965de51. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 18 22:08:44.237288 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-1e6cebf0-8316-49e5-9220-a56a7e402dbc tempest-ServerDiskConfigTestJSON-1657751511 tempest-ServerDiskConfigTestJSON-1657751511-project] [instance: 841ecf42-f966-4e34-96b8-83504a3a66e3] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120657) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 18 22:08:44.238066 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-1e6cebf0-8316-49e5-9220-a56a7e402dbc tempest-ServerDiskConfigTestJSON-1657751511 tempest-ServerDiskConfigTestJSON-1657751511-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:08:44.238509 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-1e6cebf0-8316-49e5-9220-a56a7e402dbc tempest-ServerDiskConfigTestJSON-1657751511 tempest-ServerDiskConfigTestJSON-1657751511-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:08:44.238899 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-1e6cebf0-8316-49e5-9220-a56a7e402dbc tempest-ServerDiskConfigTestJSON-1657751511 tempest-ServerDiskConfigTestJSON-1657751511-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:08:44.312139 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-1e6cebf0-8316-49e5-9220-a56a7e402dbc tempest-ServerDiskConfigTestJSON-1657751511 tempest-ServerDiskConfigTestJSON-1657751511-project] [instance: 841ecf42-f966-4e34-96b8-83504a3a66e3] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='87b0fec9-a948-41f9-a3c0-f0a6a5a8f1e7',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120657) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 18 22:08:44.313408 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-1e6cebf0-8316-49e5-9220-a56a7e402dbc tempest-ServerDiskConfigTestJSON-1657751511 tempest-ServerDiskConfigTestJSON-1657751511-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:08:44.313855 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-1e6cebf0-8316-49e5-9220-a56a7e402dbc tempest-ServerDiskConfigTestJSON-1657751511 tempest-ServerDiskConfigTestJSON-1657751511-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:08:44.314253 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-1e6cebf0-8316-49e5-9220-a56a7e402dbc tempest-ServerDiskConfigTestJSON-1657751511 tempest-ServerDiskConfigTestJSON-1657751511-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:08:44.588031 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-1e6cebf0-8316-49e5-9220-a56a7e402dbc tempest-ServerDiskConfigTestJSON-1657751511 tempest-ServerDiskConfigTestJSON-1657751511-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:08:44.588485 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-1e6cebf0-8316-49e5-9220-a56a7e402dbc tempest-ServerDiskConfigTestJSON-1657751511 tempest-ServerDiskConfigTestJSON-1657751511-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:08:44.588886 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-1e6cebf0-8316-49e5-9220-a56a7e402dbc tempest-ServerDiskConfigTestJSON-1657751511 tempest-ServerDiskConfigTestJSON-1657751511-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:08:44.589342 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-1e6cebf0-8316-49e5-9220-a56a7e402dbc tempest-ServerDiskConfigTestJSON-1657751511 tempest-ServerDiskConfigTestJSON-1657751511-project] Acquiring lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 18 22:08:44.589699 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-1e6cebf0-8316-49e5-9220-a56a7e402dbc tempest-ServerDiskConfigTestJSON-1657751511 tempest-ServerDiskConfigTestJSON-1657751511-project] Acquired lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 18 22:08:44.590507 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c30061d6-a080-43fb-84ae-5f6c5b4254e4 None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:08:44.590929 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c30061d6-a080-43fb-84ae-5f6c5b4254e4 None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:08:44.591365 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c30061d6-a080-43fb-84ae-5f6c5b4254e4 None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:08:44.592052 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c30061d6-a080-43fb-84ae-5f6c5b4254e4 None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:08:44.592492 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c30061d6-a080-43fb-84ae-5f6c5b4254e4 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:08:44.592879 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c30061d6-a080-43fb-84ae-5f6c5b4254e4 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:08:44.601686 n-d-853704-1 nova-conductor[120657]: ERROR nova.compute.rpcapi [None req-c30061d6-a080-43fb-84ae-5f6c5b4254e4 None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 18 22:08:44.602242 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c30061d6-a080-43fb-84ae-5f6c5b4254e4 None None] Releasing lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 18 22:08:44.603161 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server [None req-c30061d6-a080-43fb-84ae-5f6c5b4254e4 None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:08:44.603161 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:08:44.603161 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 18 22:08:44.603161 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 18 22:08:44.603161 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 18 22:08:44.603161 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 22:08:44.603161 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 18 22:08:44.603161 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 22:08:44.603161 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:08:44.603161 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 18 22:08:44.603161 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 18 22:08:44.603161 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 18 22:08:44.603161 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 18 22:08:44.603161 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 18 22:08:44.603161 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 18 22:08:44.603161 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 18 22:08:44.603161 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 18 22:08:44.604500 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 18 22:08:44.604500 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 18 22:08:44.604500 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 18 22:08:44.604500 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 18 22:08:44.604500 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 18 22:08:44.604500 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 18 22:08:44.604500 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:08:44.604500 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 22:08:52.990686 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=21,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:08:53.467564 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=10,nova_api:UPDATE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:08:53.557051 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=14,nova_api:DELETE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:08:53.662728 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:08:53.663744 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell1:SELECT=11,nova_cell1:SAVEPOINT=1,nova_cell1:INSERT=18,nova_cell1:RELEASE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:08:54.521107 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=11,nova_api:UPDATE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:08:54.521790 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=13,nova_api:DELETE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:08:54.597717 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:08:54.599430 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell1:SELECT=11,nova_cell1:INSERT=17 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:09:02.953971 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=7,nova_cell0:UPDATE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:09:22.955647 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=4,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:09:22.994950 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=5,nova_cell0:UPDATE=3 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:09:42.959212 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=4,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:09:43.002809 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=4,nova_cell0:UPDATE=2 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:10:03.003991 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=4,nova_cell0:UPDATE=2 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:10:22.957545 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:10:43.009839 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:11:02.960854 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:11:22.965347 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=3,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:11:23.015726 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=7,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:11:41.215847 n-d-853704-1 nova-conductor[120658]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 18 22:12:02.959778 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:12:03.010354 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:12:32.963214 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=5,nova_cell0:UPDATE=3 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:12:43.000677 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:12:45.321459 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-505e21f3-86cd-4d7e-9383-09812c1c8bf9 tempest-LiveMigrationNegativeTest-1198353743 tempest-LiveMigrationNegativeTest-1198353743-project] Took 0.52 seconds to select destinations for 1 instance(s). {{(pid=120658) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 18 22:12:45.340403 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-505e21f3-86cd-4d7e-9383-09812c1c8bf9 tempest-LiveMigrationNegativeTest-1198353743 tempest-LiveMigrationNegativeTest-1198353743-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:12:45.340848 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-505e21f3-86cd-4d7e-9383-09812c1c8bf9 tempest-LiveMigrationNegativeTest-1198353743 tempest-LiveMigrationNegativeTest-1198353743-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:12:45.341248 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-505e21f3-86cd-4d7e-9383-09812c1c8bf9 tempest-LiveMigrationNegativeTest-1198353743 tempest-LiveMigrationNegativeTest-1198353743-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:12:45.496598 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-505e21f3-86cd-4d7e-9383-09812c1c8bf9 tempest-LiveMigrationNegativeTest-1198353743 tempest-LiveMigrationNegativeTest-1198353743-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:12:45.497032 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-505e21f3-86cd-4d7e-9383-09812c1c8bf9 tempest-LiveMigrationNegativeTest-1198353743 tempest-LiveMigrationNegativeTest-1198353743-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:12:45.497458 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-505e21f3-86cd-4d7e-9383-09812c1c8bf9 tempest-LiveMigrationNegativeTest-1198353743 tempest-LiveMigrationNegativeTest-1198353743-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:12:45.498109 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-505e21f3-86cd-4d7e-9383-09812c1c8bf9 tempest-LiveMigrationNegativeTest-1198353743 tempest-LiveMigrationNegativeTest-1198353743-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:12:45.498516 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-505e21f3-86cd-4d7e-9383-09812c1c8bf9 tempest-LiveMigrationNegativeTest-1198353743 tempest-LiveMigrationNegativeTest-1198353743-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:12:45.498898 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-505e21f3-86cd-4d7e-9383-09812c1c8bf9 tempest-LiveMigrationNegativeTest-1198353743 tempest-LiveMigrationNegativeTest-1198353743-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:12:45.514486 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-505e21f3-86cd-4d7e-9383-09812c1c8bf9 tempest-LiveMigrationNegativeTest-1198353743 tempest-LiveMigrationNegativeTest-1198353743-project] Getting quotas for project 5e5533ee845b4a1290b8fc32a3030755. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 18 22:12:45.518426 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-505e21f3-86cd-4d7e-9383-09812c1c8bf9 tempest-LiveMigrationNegativeTest-1198353743 tempest-LiveMigrationNegativeTest-1198353743-project] Getting quotas for user cbf8eb9046654b68bd8be43f6570cee9 and project 5e5533ee845b4a1290b8fc32a3030755. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 18 22:12:45.529565 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-505e21f3-86cd-4d7e-9383-09812c1c8bf9 tempest-LiveMigrationNegativeTest-1198353743 tempest-LiveMigrationNegativeTest-1198353743-project] [instance: 5d4c318d-3000-4286-b0fe-de1d177a4571] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120658) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 18 22:12:45.530315 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-505e21f3-86cd-4d7e-9383-09812c1c8bf9 tempest-LiveMigrationNegativeTest-1198353743 tempest-LiveMigrationNegativeTest-1198353743-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:12:45.530744 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-505e21f3-86cd-4d7e-9383-09812c1c8bf9 tempest-LiveMigrationNegativeTest-1198353743 tempest-LiveMigrationNegativeTest-1198353743-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:12:45.531129 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-505e21f3-86cd-4d7e-9383-09812c1c8bf9 tempest-LiveMigrationNegativeTest-1198353743 tempest-LiveMigrationNegativeTest-1198353743-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:12:45.610028 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-505e21f3-86cd-4d7e-9383-09812c1c8bf9 tempest-LiveMigrationNegativeTest-1198353743 tempest-LiveMigrationNegativeTest-1198353743-project] [instance: 5d4c318d-3000-4286-b0fe-de1d177a4571] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='87b0fec9-a948-41f9-a3c0-f0a6a5a8f1e7',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120658) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 18 22:12:45.611238 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-505e21f3-86cd-4d7e-9383-09812c1c8bf9 tempest-LiveMigrationNegativeTest-1198353743 tempest-LiveMigrationNegativeTest-1198353743-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:12:45.611690 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-505e21f3-86cd-4d7e-9383-09812c1c8bf9 tempest-LiveMigrationNegativeTest-1198353743 tempest-LiveMigrationNegativeTest-1198353743-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:12:45.612108 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-505e21f3-86cd-4d7e-9383-09812c1c8bf9 tempest-LiveMigrationNegativeTest-1198353743 tempest-LiveMigrationNegativeTest-1198353743-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:12:45.844846 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-505e21f3-86cd-4d7e-9383-09812c1c8bf9 tempest-LiveMigrationNegativeTest-1198353743 tempest-LiveMigrationNegativeTest-1198353743-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:12:45.845281 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-505e21f3-86cd-4d7e-9383-09812c1c8bf9 tempest-LiveMigrationNegativeTest-1198353743 tempest-LiveMigrationNegativeTest-1198353743-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:12:45.845672 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-505e21f3-86cd-4d7e-9383-09812c1c8bf9 tempest-LiveMigrationNegativeTest-1198353743 tempest-LiveMigrationNegativeTest-1198353743-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:12:45.846119 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-505e21f3-86cd-4d7e-9383-09812c1c8bf9 tempest-LiveMigrationNegativeTest-1198353743 tempest-LiveMigrationNegativeTest-1198353743-project] Acquiring lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 18 22:12:45.846467 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-505e21f3-86cd-4d7e-9383-09812c1c8bf9 tempest-LiveMigrationNegativeTest-1198353743 tempest-LiveMigrationNegativeTest-1198353743-project] Acquired lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 18 22:12:45.847185 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-3631eba5-2ded-443b-982b-41d070bb75c6 None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:12:45.847597 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-3631eba5-2ded-443b-982b-41d070bb75c6 None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:12:45.848001 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-3631eba5-2ded-443b-982b-41d070bb75c6 None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:12:45.848638 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-3631eba5-2ded-443b-982b-41d070bb75c6 None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:12:45.849040 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-3631eba5-2ded-443b-982b-41d070bb75c6 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:12:45.849415 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-3631eba5-2ded-443b-982b-41d070bb75c6 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:12:45.857489 n-d-853704-1 nova-conductor[120658]: ERROR nova.compute.rpcapi [None req-3631eba5-2ded-443b-982b-41d070bb75c6 None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 18 22:12:45.857868 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-3631eba5-2ded-443b-982b-41d070bb75c6 None None] Releasing lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 18 22:12:45.858757 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server [None req-3631eba5-2ded-443b-982b-41d070bb75c6 None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:12:45.858757 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:12:45.858757 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 18 22:12:45.858757 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 18 22:12:45.858757 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 18 22:12:45.858757 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 22:12:45.858757 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 18 22:12:45.858757 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 22:12:45.858757 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:12:45.858757 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 18 22:12:45.858757 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 18 22:12:45.858757 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 18 22:12:45.858757 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 18 22:12:45.858757 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 18 22:12:45.858757 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 18 22:12:45.858757 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 18 22:12:45.858757 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 18 22:12:45.860168 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 18 22:12:45.860168 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 18 22:12:45.860168 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 18 22:12:45.860168 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 18 22:12:45.860168 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 18 22:12:45.860168 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 18 22:12:45.860168 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:12:45.860168 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 22:12:55.698853 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=10,nova_api:UPDATE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:12:55.778835 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=14,nova_api:DELETE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:12:55.855430 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:12:55.861938 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell1:SELECT=11,nova_cell1:INSERT=17 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:13:12.966211 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:13:23.009357 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=13,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:13:29.197067 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-172477f9-fe69-41eb-9b90-723fa6f57d60 tempest-ImagesTestJSON-1761366649 tempest-ImagesTestJSON-1761366649-project] Took 0.36 seconds to select destinations for 1 instance(s). {{(pid=120657) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 18 22:13:29.217399 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-172477f9-fe69-41eb-9b90-723fa6f57d60 tempest-ImagesTestJSON-1761366649 tempest-ImagesTestJSON-1761366649-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:13:29.217955 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-172477f9-fe69-41eb-9b90-723fa6f57d60 tempest-ImagesTestJSON-1761366649 tempest-ImagesTestJSON-1761366649-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:13:29.218422 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-172477f9-fe69-41eb-9b90-723fa6f57d60 tempest-ImagesTestJSON-1761366649 tempest-ImagesTestJSON-1761366649-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:13:29.346631 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-172477f9-fe69-41eb-9b90-723fa6f57d60 tempest-ImagesTestJSON-1761366649 tempest-ImagesTestJSON-1761366649-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:13:29.347084 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-172477f9-fe69-41eb-9b90-723fa6f57d60 tempest-ImagesTestJSON-1761366649 tempest-ImagesTestJSON-1761366649-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:13:29.347485 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-172477f9-fe69-41eb-9b90-723fa6f57d60 tempest-ImagesTestJSON-1761366649 tempest-ImagesTestJSON-1761366649-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:13:29.348186 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-172477f9-fe69-41eb-9b90-723fa6f57d60 tempest-ImagesTestJSON-1761366649 tempest-ImagesTestJSON-1761366649-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:13:29.348607 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-172477f9-fe69-41eb-9b90-723fa6f57d60 tempest-ImagesTestJSON-1761366649 tempest-ImagesTestJSON-1761366649-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:13:29.348994 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-172477f9-fe69-41eb-9b90-723fa6f57d60 tempest-ImagesTestJSON-1761366649 tempest-ImagesTestJSON-1761366649-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:13:29.365362 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-172477f9-fe69-41eb-9b90-723fa6f57d60 tempest-ImagesTestJSON-1761366649 tempest-ImagesTestJSON-1761366649-project] Getting quotas for project 677821f62eb848bcaf47c31a724218df. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 18 22:13:29.369392 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-172477f9-fe69-41eb-9b90-723fa6f57d60 tempest-ImagesTestJSON-1761366649 tempest-ImagesTestJSON-1761366649-project] Getting quotas for user 2b6fddb708684c6a8d541e617aa2fc48 and project 677821f62eb848bcaf47c31a724218df. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 18 22:13:29.381088 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-172477f9-fe69-41eb-9b90-723fa6f57d60 tempest-ImagesTestJSON-1761366649 tempest-ImagesTestJSON-1761366649-project] [instance: c1bbab8a-b0c7-4796-88f9-9d8aaa350d01] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120657) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 18 22:13:29.381889 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-172477f9-fe69-41eb-9b90-723fa6f57d60 tempest-ImagesTestJSON-1761366649 tempest-ImagesTestJSON-1761366649-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:13:29.382314 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-172477f9-fe69-41eb-9b90-723fa6f57d60 tempest-ImagesTestJSON-1761366649 tempest-ImagesTestJSON-1761366649-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:13:29.382716 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-172477f9-fe69-41eb-9b90-723fa6f57d60 tempest-ImagesTestJSON-1761366649 tempest-ImagesTestJSON-1761366649-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:13:29.441755 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-172477f9-fe69-41eb-9b90-723fa6f57d60 tempest-ImagesTestJSON-1761366649 tempest-ImagesTestJSON-1761366649-project] [instance: c1bbab8a-b0c7-4796-88f9-9d8aaa350d01] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='87b0fec9-a948-41f9-a3c0-f0a6a5a8f1e7',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120657) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 18 22:13:29.442979 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-172477f9-fe69-41eb-9b90-723fa6f57d60 tempest-ImagesTestJSON-1761366649 tempest-ImagesTestJSON-1761366649-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:13:29.443419 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-172477f9-fe69-41eb-9b90-723fa6f57d60 tempest-ImagesTestJSON-1761366649 tempest-ImagesTestJSON-1761366649-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:13:29.443849 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-172477f9-fe69-41eb-9b90-723fa6f57d60 tempest-ImagesTestJSON-1761366649 tempest-ImagesTestJSON-1761366649-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:13:29.682076 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-172477f9-fe69-41eb-9b90-723fa6f57d60 tempest-ImagesTestJSON-1761366649 tempest-ImagesTestJSON-1761366649-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:13:29.682485 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-172477f9-fe69-41eb-9b90-723fa6f57d60 tempest-ImagesTestJSON-1761366649 tempest-ImagesTestJSON-1761366649-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:13:29.682877 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-172477f9-fe69-41eb-9b90-723fa6f57d60 tempest-ImagesTestJSON-1761366649 tempest-ImagesTestJSON-1761366649-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:13:29.683322 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-172477f9-fe69-41eb-9b90-723fa6f57d60 tempest-ImagesTestJSON-1761366649 tempest-ImagesTestJSON-1761366649-project] Acquiring lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 18 22:13:29.683711 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-172477f9-fe69-41eb-9b90-723fa6f57d60 tempest-ImagesTestJSON-1761366649 tempest-ImagesTestJSON-1761366649-project] Acquired lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 18 22:13:29.685100 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-367780e7-8444-481f-977b-a08e7ab0cc7c None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:13:29.685516 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-367780e7-8444-481f-977b-a08e7ab0cc7c None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:13:29.685899 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-367780e7-8444-481f-977b-a08e7ab0cc7c None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:13:29.686537 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-367780e7-8444-481f-977b-a08e7ab0cc7c None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:13:29.686941 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-367780e7-8444-481f-977b-a08e7ab0cc7c None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:13:29.687316 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-367780e7-8444-481f-977b-a08e7ab0cc7c None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:13:29.695549 n-d-853704-1 nova-conductor[120657]: ERROR nova.compute.rpcapi [None req-367780e7-8444-481f-977b-a08e7ab0cc7c None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 18 22:13:29.695956 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-367780e7-8444-481f-977b-a08e7ab0cc7c None None] Releasing lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 18 22:13:29.696853 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server [None req-367780e7-8444-481f-977b-a08e7ab0cc7c None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:13:29.696853 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:13:29.696853 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 18 22:13:29.696853 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 18 22:13:29.696853 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 18 22:13:29.696853 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 22:13:29.696853 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 18 22:13:29.696853 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 22:13:29.696853 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:13:29.696853 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 18 22:13:29.696853 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 18 22:13:29.696853 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 18 22:13:29.696853 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 18 22:13:29.696853 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 18 22:13:29.696853 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 18 22:13:29.696853 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 18 22:13:29.696853 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 18 22:13:29.698255 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 18 22:13:29.698255 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 18 22:13:29.698255 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 18 22:13:29.698255 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 18 22:13:29.698255 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 18 22:13:29.698255 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 18 22:13:29.698255 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:13:29.698255 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 22:13:39.625918 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=12,nova_api:UPDATE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:13:39.627091 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=12,nova_api:DELETE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:13:39.693142 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:13:39.693760 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell1:SELECT=11,nova_cell1:INSERT=17 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:13:52.967102 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=14,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:14:03.009493 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:14:14.904581 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-ba2bd21e-57d1-4960-8959-0309423ab4d2 tempest-MigrationsAdminTest-1637837745 tempest-MigrationsAdminTest-1637837745-project] Took 0.61 seconds to select destinations for 1 instance(s). {{(pid=120658) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 18 22:14:14.919294 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ba2bd21e-57d1-4960-8959-0309423ab4d2 tempest-MigrationsAdminTest-1637837745 tempest-MigrationsAdminTest-1637837745-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:14:14.919926 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ba2bd21e-57d1-4960-8959-0309423ab4d2 tempest-MigrationsAdminTest-1637837745 tempest-MigrationsAdminTest-1637837745-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:14:14.920168 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ba2bd21e-57d1-4960-8959-0309423ab4d2 tempest-MigrationsAdminTest-1637837745 tempest-MigrationsAdminTest-1637837745-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:14:15.151482 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ba2bd21e-57d1-4960-8959-0309423ab4d2 tempest-MigrationsAdminTest-1637837745 tempest-MigrationsAdminTest-1637837745-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:14:15.151954 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ba2bd21e-57d1-4960-8959-0309423ab4d2 tempest-MigrationsAdminTest-1637837745 tempest-MigrationsAdminTest-1637837745-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:14:15.152349 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ba2bd21e-57d1-4960-8959-0309423ab4d2 tempest-MigrationsAdminTest-1637837745 tempest-MigrationsAdminTest-1637837745-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:14:15.153009 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ba2bd21e-57d1-4960-8959-0309423ab4d2 tempest-MigrationsAdminTest-1637837745 tempest-MigrationsAdminTest-1637837745-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:14:15.153423 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ba2bd21e-57d1-4960-8959-0309423ab4d2 tempest-MigrationsAdminTest-1637837745 tempest-MigrationsAdminTest-1637837745-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:14:15.153804 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ba2bd21e-57d1-4960-8959-0309423ab4d2 tempest-MigrationsAdminTest-1637837745 tempest-MigrationsAdminTest-1637837745-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:14:15.169532 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-ba2bd21e-57d1-4960-8959-0309423ab4d2 tempest-MigrationsAdminTest-1637837745 tempest-MigrationsAdminTest-1637837745-project] Getting quotas for project 55cd873dadd6496dbebea5c3290775eb. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 18 22:14:15.174684 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-ba2bd21e-57d1-4960-8959-0309423ab4d2 tempest-MigrationsAdminTest-1637837745 tempest-MigrationsAdminTest-1637837745-project] Getting quotas for user caa4813c18704375b16dedceb822364f and project 55cd873dadd6496dbebea5c3290775eb. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 18 22:14:15.185196 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-ba2bd21e-57d1-4960-8959-0309423ab4d2 tempest-MigrationsAdminTest-1637837745 tempest-MigrationsAdminTest-1637837745-project] [instance: 791e2a02-776b-4a8f-8041-8ea39a8038bb] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120658) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 18 22:14:15.185978 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ba2bd21e-57d1-4960-8959-0309423ab4d2 tempest-MigrationsAdminTest-1637837745 tempest-MigrationsAdminTest-1637837745-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:14:15.186415 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ba2bd21e-57d1-4960-8959-0309423ab4d2 tempest-MigrationsAdminTest-1637837745 tempest-MigrationsAdminTest-1637837745-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:14:15.186809 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ba2bd21e-57d1-4960-8959-0309423ab4d2 tempest-MigrationsAdminTest-1637837745 tempest-MigrationsAdminTest-1637837745-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:14:15.269404 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-ba2bd21e-57d1-4960-8959-0309423ab4d2 tempest-MigrationsAdminTest-1637837745 tempest-MigrationsAdminTest-1637837745-project] [instance: 791e2a02-776b-4a8f-8041-8ea39a8038bb] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='87b0fec9-a948-41f9-a3c0-f0a6a5a8f1e7',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120658) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 18 22:14:15.270654 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ba2bd21e-57d1-4960-8959-0309423ab4d2 tempest-MigrationsAdminTest-1637837745 tempest-MigrationsAdminTest-1637837745-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:14:15.271126 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ba2bd21e-57d1-4960-8959-0309423ab4d2 tempest-MigrationsAdminTest-1637837745 tempest-MigrationsAdminTest-1637837745-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:14:15.271518 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ba2bd21e-57d1-4960-8959-0309423ab4d2 tempest-MigrationsAdminTest-1637837745 tempest-MigrationsAdminTest-1637837745-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:14:15.509838 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ba2bd21e-57d1-4960-8959-0309423ab4d2 tempest-MigrationsAdminTest-1637837745 tempest-MigrationsAdminTest-1637837745-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:14:15.510283 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ba2bd21e-57d1-4960-8959-0309423ab4d2 tempest-MigrationsAdminTest-1637837745 tempest-MigrationsAdminTest-1637837745-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:14:15.510720 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ba2bd21e-57d1-4960-8959-0309423ab4d2 tempest-MigrationsAdminTest-1637837745 tempest-MigrationsAdminTest-1637837745-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:14:15.511206 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ba2bd21e-57d1-4960-8959-0309423ab4d2 tempest-MigrationsAdminTest-1637837745 tempest-MigrationsAdminTest-1637837745-project] Acquiring lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 18 22:14:15.516160 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ba2bd21e-57d1-4960-8959-0309423ab4d2 tempest-MigrationsAdminTest-1637837745 tempest-MigrationsAdminTest-1637837745-project] Acquired lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 18 22:14:15.516160 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-eb71006c-b72a-49e7-88be-01ab76e44a73 None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:14:15.516160 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-eb71006c-b72a-49e7-88be-01ab76e44a73 None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:14:15.516160 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-eb71006c-b72a-49e7-88be-01ab76e44a73 None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:14:15.516160 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-eb71006c-b72a-49e7-88be-01ab76e44a73 None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:14:15.516663 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-eb71006c-b72a-49e7-88be-01ab76e44a73 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:14:15.516663 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-eb71006c-b72a-49e7-88be-01ab76e44a73 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:14:15.523411 n-d-853704-1 nova-conductor[120658]: ERROR nova.compute.rpcapi [None req-eb71006c-b72a-49e7-88be-01ab76e44a73 None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 18 22:14:15.523839 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-eb71006c-b72a-49e7-88be-01ab76e44a73 None None] Releasing lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 18 22:14:15.524752 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server [None req-eb71006c-b72a-49e7-88be-01ab76e44a73 None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:14:15.524752 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:14:15.524752 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 18 22:14:15.524752 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 18 22:14:15.524752 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 18 22:14:15.524752 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 22:14:15.524752 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 18 22:14:15.524752 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 22:14:15.524752 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:14:15.524752 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 18 22:14:15.524752 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 18 22:14:15.524752 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 18 22:14:15.524752 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 18 22:14:15.524752 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 18 22:14:15.524752 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 18 22:14:15.524752 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 18 22:14:15.524752 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 18 22:14:15.526143 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 18 22:14:15.526143 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 18 22:14:15.526143 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 18 22:14:15.526143 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 18 22:14:15.526143 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 18 22:14:15.526143 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 18 22:14:15.526143 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:14:15.526143 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 22:14:25.432504 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=12,nova_api:UPDATE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:14:25.433393 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=12,nova_api:DELETE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:14:25.520524 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:14:25.521296 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell1:SELECT=11,nova_cell1:SAVEPOINT=1,nova_cell1:INSERT=18,nova_cell1:RELEASE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:14:32.978050 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:14:43.021652 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=13,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:14:43.644076 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-9838021c-a0da-4ec3-b617-5339ccbe7343 tempest-SecurityGroupsTestJSON-433475947 tempest-SecurityGroupsTestJSON-433475947-project] Took 0.93 seconds to select destinations for 1 instance(s). {{(pid=120657) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 18 22:14:43.672310 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-9838021c-a0da-4ec3-b617-5339ccbe7343 tempest-SecurityGroupsTestJSON-433475947 tempest-SecurityGroupsTestJSON-433475947-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:14:43.673268 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-9838021c-a0da-4ec3-b617-5339ccbe7343 tempest-SecurityGroupsTestJSON-433475947 tempest-SecurityGroupsTestJSON-433475947-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:14:43.673822 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-9838021c-a0da-4ec3-b617-5339ccbe7343 tempest-SecurityGroupsTestJSON-433475947 tempest-SecurityGroupsTestJSON-433475947-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:14:43.939392 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-9838021c-a0da-4ec3-b617-5339ccbe7343 tempest-SecurityGroupsTestJSON-433475947 tempest-SecurityGroupsTestJSON-433475947-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:14:43.940299 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-9838021c-a0da-4ec3-b617-5339ccbe7343 tempest-SecurityGroupsTestJSON-433475947 tempest-SecurityGroupsTestJSON-433475947-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:14:43.940699 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-9838021c-a0da-4ec3-b617-5339ccbe7343 tempest-SecurityGroupsTestJSON-433475947 tempest-SecurityGroupsTestJSON-433475947-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:14:43.941377 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-9838021c-a0da-4ec3-b617-5339ccbe7343 tempest-SecurityGroupsTestJSON-433475947 tempest-SecurityGroupsTestJSON-433475947-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:14:43.941821 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-9838021c-a0da-4ec3-b617-5339ccbe7343 tempest-SecurityGroupsTestJSON-433475947 tempest-SecurityGroupsTestJSON-433475947-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:14:43.942173 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-9838021c-a0da-4ec3-b617-5339ccbe7343 tempest-SecurityGroupsTestJSON-433475947 tempest-SecurityGroupsTestJSON-433475947-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:14:43.968377 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-9838021c-a0da-4ec3-b617-5339ccbe7343 tempest-SecurityGroupsTestJSON-433475947 tempest-SecurityGroupsTestJSON-433475947-project] Getting quotas for project f1a2a950c02b41c08f58cda4453a2865. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 18 22:14:43.974096 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-9838021c-a0da-4ec3-b617-5339ccbe7343 tempest-SecurityGroupsTestJSON-433475947 tempest-SecurityGroupsTestJSON-433475947-project] Getting quotas for user 8e8024c8bb464e189bc0ac5069891bb9 and project f1a2a950c02b41c08f58cda4453a2865. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 18 22:14:43.992844 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-9838021c-a0da-4ec3-b617-5339ccbe7343 tempest-SecurityGroupsTestJSON-433475947 tempest-SecurityGroupsTestJSON-433475947-project] [instance: bad195ec-ac6e-4bf3-87d4-aff4986a6c83] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120657) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 18 22:14:43.994067 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-9838021c-a0da-4ec3-b617-5339ccbe7343 tempest-SecurityGroupsTestJSON-433475947 tempest-SecurityGroupsTestJSON-433475947-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:14:43.994507 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-9838021c-a0da-4ec3-b617-5339ccbe7343 tempest-SecurityGroupsTestJSON-433475947 tempest-SecurityGroupsTestJSON-433475947-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:14:43.994904 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-9838021c-a0da-4ec3-b617-5339ccbe7343 tempest-SecurityGroupsTestJSON-433475947 tempest-SecurityGroupsTestJSON-433475947-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:14:44.088998 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-9838021c-a0da-4ec3-b617-5339ccbe7343 tempest-SecurityGroupsTestJSON-433475947 tempest-SecurityGroupsTestJSON-433475947-project] [instance: bad195ec-ac6e-4bf3-87d4-aff4986a6c83] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='87b0fec9-a948-41f9-a3c0-f0a6a5a8f1e7',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120657) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 18 22:14:44.094386 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-9838021c-a0da-4ec3-b617-5339ccbe7343 tempest-SecurityGroupsTestJSON-433475947 tempest-SecurityGroupsTestJSON-433475947-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:14:44.094386 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-9838021c-a0da-4ec3-b617-5339ccbe7343 tempest-SecurityGroupsTestJSON-433475947 tempest-SecurityGroupsTestJSON-433475947-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:14:44.094386 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-9838021c-a0da-4ec3-b617-5339ccbe7343 tempest-SecurityGroupsTestJSON-433475947 tempest-SecurityGroupsTestJSON-433475947-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:14:44.457023 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-9838021c-a0da-4ec3-b617-5339ccbe7343 tempest-SecurityGroupsTestJSON-433475947 tempest-SecurityGroupsTestJSON-433475947-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:14:44.457583 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-9838021c-a0da-4ec3-b617-5339ccbe7343 tempest-SecurityGroupsTestJSON-433475947 tempest-SecurityGroupsTestJSON-433475947-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:14:44.458126 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-9838021c-a0da-4ec3-b617-5339ccbe7343 tempest-SecurityGroupsTestJSON-433475947 tempest-SecurityGroupsTestJSON-433475947-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:14:44.459399 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-9838021c-a0da-4ec3-b617-5339ccbe7343 tempest-SecurityGroupsTestJSON-433475947 tempest-SecurityGroupsTestJSON-433475947-project] Acquiring lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 18 22:14:44.465684 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-9838021c-a0da-4ec3-b617-5339ccbe7343 tempest-SecurityGroupsTestJSON-433475947 tempest-SecurityGroupsTestJSON-433475947-project] Acquired lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 18 22:14:44.465684 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-0d93b8a0-f49e-4d76-ba0f-003086f0f47d None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:14:44.465684 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-0d93b8a0-f49e-4d76-ba0f-003086f0f47d None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:14:44.465684 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-0d93b8a0-f49e-4d76-ba0f-003086f0f47d None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:14:44.465684 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-0d93b8a0-f49e-4d76-ba0f-003086f0f47d None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:14:44.466169 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-0d93b8a0-f49e-4d76-ba0f-003086f0f47d None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:14:44.466169 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-0d93b8a0-f49e-4d76-ba0f-003086f0f47d None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:14:44.491168 n-d-853704-1 nova-conductor[120657]: ERROR nova.compute.rpcapi [None req-0d93b8a0-f49e-4d76-ba0f-003086f0f47d None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 18 22:14:44.493010 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-0d93b8a0-f49e-4d76-ba0f-003086f0f47d None None] Releasing lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 18 22:14:44.494404 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server [None req-0d93b8a0-f49e-4d76-ba0f-003086f0f47d None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:14:44.494404 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:14:44.494404 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 18 22:14:44.494404 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 18 22:14:44.494404 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 18 22:14:44.494404 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 22:14:44.494404 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 18 22:14:44.494404 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 22:14:44.494404 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:14:44.494404 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 18 22:14:44.494404 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 18 22:14:44.494404 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 18 22:14:44.494404 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 18 22:14:44.494404 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 18 22:14:44.494404 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 18 22:14:44.494404 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 18 22:14:44.494404 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 18 22:14:44.495711 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 18 22:14:44.495711 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 18 22:14:44.495711 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 18 22:14:44.495711 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 18 22:14:44.495711 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 18 22:14:44.495711 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 18 22:14:44.495711 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:14:44.495711 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 22:14:48.017994 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-3ed94851-9f2c-416e-a427-265417b84bfc tempest-ServerDiskConfigTestJSON-1657751511 tempest-ServerDiskConfigTestJSON-1657751511-project] Took 0.67 seconds to select destinations for 1 instance(s). {{(pid=120658) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 18 22:14:48.074058 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-3ed94851-9f2c-416e-a427-265417b84bfc tempest-ServerDiskConfigTestJSON-1657751511 tempest-ServerDiskConfigTestJSON-1657751511-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:14:48.074508 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-3ed94851-9f2c-416e-a427-265417b84bfc tempest-ServerDiskConfigTestJSON-1657751511 tempest-ServerDiskConfigTestJSON-1657751511-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:14:48.079329 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-3ed94851-9f2c-416e-a427-265417b84bfc tempest-ServerDiskConfigTestJSON-1657751511 tempest-ServerDiskConfigTestJSON-1657751511-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:14:48.226651 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-3ed94851-9f2c-416e-a427-265417b84bfc tempest-ServerDiskConfigTestJSON-1657751511 tempest-ServerDiskConfigTestJSON-1657751511-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:14:48.227469 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-3ed94851-9f2c-416e-a427-265417b84bfc tempest-ServerDiskConfigTestJSON-1657751511 tempest-ServerDiskConfigTestJSON-1657751511-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:14:48.228429 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-3ed94851-9f2c-416e-a427-265417b84bfc tempest-ServerDiskConfigTestJSON-1657751511 tempest-ServerDiskConfigTestJSON-1657751511-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:14:48.229637 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-3ed94851-9f2c-416e-a427-265417b84bfc tempest-ServerDiskConfigTestJSON-1657751511 tempest-ServerDiskConfigTestJSON-1657751511-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:14:48.230292 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-3ed94851-9f2c-416e-a427-265417b84bfc tempest-ServerDiskConfigTestJSON-1657751511 tempest-ServerDiskConfigTestJSON-1657751511-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:14:48.231605 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-3ed94851-9f2c-416e-a427-265417b84bfc tempest-ServerDiskConfigTestJSON-1657751511 tempest-ServerDiskConfigTestJSON-1657751511-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:14:48.254015 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-3ed94851-9f2c-416e-a427-265417b84bfc tempest-ServerDiskConfigTestJSON-1657751511 tempest-ServerDiskConfigTestJSON-1657751511-project] Getting quotas for project 4450d6767020433e986d18bb2965de51. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 18 22:14:48.258795 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-3ed94851-9f2c-416e-a427-265417b84bfc tempest-ServerDiskConfigTestJSON-1657751511 tempest-ServerDiskConfigTestJSON-1657751511-project] Getting quotas for user 9a6f17cb5745439882594505d7f20df1 and project 4450d6767020433e986d18bb2965de51. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 18 22:14:48.273066 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-3ed94851-9f2c-416e-a427-265417b84bfc tempest-ServerDiskConfigTestJSON-1657751511 tempest-ServerDiskConfigTestJSON-1657751511-project] [instance: e88f90e6-7770-45f9-9ff9-234863cd6d15] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120658) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 18 22:14:48.274246 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-3ed94851-9f2c-416e-a427-265417b84bfc tempest-ServerDiskConfigTestJSON-1657751511 tempest-ServerDiskConfigTestJSON-1657751511-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:14:48.274714 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-3ed94851-9f2c-416e-a427-265417b84bfc tempest-ServerDiskConfigTestJSON-1657751511 tempest-ServerDiskConfigTestJSON-1657751511-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:14:48.275120 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-3ed94851-9f2c-416e-a427-265417b84bfc tempest-ServerDiskConfigTestJSON-1657751511 tempest-ServerDiskConfigTestJSON-1657751511-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:14:48.374495 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-3ed94851-9f2c-416e-a427-265417b84bfc tempest-ServerDiskConfigTestJSON-1657751511 tempest-ServerDiskConfigTestJSON-1657751511-project] [instance: e88f90e6-7770-45f9-9ff9-234863cd6d15] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='87b0fec9-a948-41f9-a3c0-f0a6a5a8f1e7',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120658) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 18 22:14:48.375761 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-3ed94851-9f2c-416e-a427-265417b84bfc tempest-ServerDiskConfigTestJSON-1657751511 tempest-ServerDiskConfigTestJSON-1657751511-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:14:48.376807 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-3ed94851-9f2c-416e-a427-265417b84bfc tempest-ServerDiskConfigTestJSON-1657751511 tempest-ServerDiskConfigTestJSON-1657751511-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:14:48.377147 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-3ed94851-9f2c-416e-a427-265417b84bfc tempest-ServerDiskConfigTestJSON-1657751511 tempest-ServerDiskConfigTestJSON-1657751511-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:14:48.672800 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-3ed94851-9f2c-416e-a427-265417b84bfc tempest-ServerDiskConfigTestJSON-1657751511 tempest-ServerDiskConfigTestJSON-1657751511-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:14:48.680994 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-3ed94851-9f2c-416e-a427-265417b84bfc tempest-ServerDiskConfigTestJSON-1657751511 tempest-ServerDiskConfigTestJSON-1657751511-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:14:48.681643 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-3ed94851-9f2c-416e-a427-265417b84bfc tempest-ServerDiskConfigTestJSON-1657751511 tempest-ServerDiskConfigTestJSON-1657751511-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.007s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:14:48.682230 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-3ed94851-9f2c-416e-a427-265417b84bfc tempest-ServerDiskConfigTestJSON-1657751511 tempest-ServerDiskConfigTestJSON-1657751511-project] Acquiring lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 18 22:14:48.682634 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-3ed94851-9f2c-416e-a427-265417b84bfc tempest-ServerDiskConfigTestJSON-1657751511 tempest-ServerDiskConfigTestJSON-1657751511-project] Acquired lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 18 22:14:48.687906 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-78a32568-9ac6-4f37-b00b-37be9c287ec7 None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:14:48.687906 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-78a32568-9ac6-4f37-b00b-37be9c287ec7 None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.002s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:14:48.687906 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-78a32568-9ac6-4f37-b00b-37be9c287ec7 None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:14:48.687906 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-78a32568-9ac6-4f37-b00b-37be9c287ec7 None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:14:48.687906 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-78a32568-9ac6-4f37-b00b-37be9c287ec7 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:14:48.689195 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-78a32568-9ac6-4f37-b00b-37be9c287ec7 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:14:48.707931 n-d-853704-1 nova-conductor[120658]: ERROR nova.compute.rpcapi [None req-78a32568-9ac6-4f37-b00b-37be9c287ec7 None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 18 22:14:48.708592 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-78a32568-9ac6-4f37-b00b-37be9c287ec7 None None] Releasing lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 18 22:14:48.710613 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server [None req-78a32568-9ac6-4f37-b00b-37be9c287ec7 None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:14:48.710613 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:14:48.710613 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 18 22:14:48.710613 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 18 22:14:48.710613 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 18 22:14:48.710613 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 22:14:48.710613 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 18 22:14:48.710613 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 22:14:48.710613 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:14:48.710613 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 18 22:14:48.710613 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 18 22:14:48.710613 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 18 22:14:48.710613 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 18 22:14:48.710613 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 18 22:14:48.710613 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 18 22:14:48.710613 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 18 22:14:48.710613 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 18 22:14:48.716778 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 18 22:14:48.716778 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 18 22:14:48.716778 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 18 22:14:48.716778 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 18 22:14:48.716778 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 18 22:14:48.716778 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 18 22:14:48.716778 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:14:48.716778 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 22:14:52.067836 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-b948de81-f7d3-42d6-8f78-27acd7b96b95 tempest-ServerAddressesTestJSON-2064647410 tempest-ServerAddressesTestJSON-2064647410-project] Took 0.68 seconds to select destinations for 1 instance(s). {{(pid=120657) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 18 22:14:52.092107 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-b948de81-f7d3-42d6-8f78-27acd7b96b95 tempest-ServerAddressesTestJSON-2064647410 tempest-ServerAddressesTestJSON-2064647410-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:14:52.092562 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-b948de81-f7d3-42d6-8f78-27acd7b96b95 tempest-ServerAddressesTestJSON-2064647410 tempest-ServerAddressesTestJSON-2064647410-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:14:52.093034 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-b948de81-f7d3-42d6-8f78-27acd7b96b95 tempest-ServerAddressesTestJSON-2064647410 tempest-ServerAddressesTestJSON-2064647410-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:14:52.300371 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-b948de81-f7d3-42d6-8f78-27acd7b96b95 tempest-ServerAddressesTestJSON-2064647410 tempest-ServerAddressesTestJSON-2064647410-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:14:52.300802 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-b948de81-f7d3-42d6-8f78-27acd7b96b95 tempest-ServerAddressesTestJSON-2064647410 tempest-ServerAddressesTestJSON-2064647410-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:14:52.301197 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-b948de81-f7d3-42d6-8f78-27acd7b96b95 tempest-ServerAddressesTestJSON-2064647410 tempest-ServerAddressesTestJSON-2064647410-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:14:52.302106 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-b948de81-f7d3-42d6-8f78-27acd7b96b95 tempest-ServerAddressesTestJSON-2064647410 tempest-ServerAddressesTestJSON-2064647410-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:14:52.302530 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-b948de81-f7d3-42d6-8f78-27acd7b96b95 tempest-ServerAddressesTestJSON-2064647410 tempest-ServerAddressesTestJSON-2064647410-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:14:52.302946 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-b948de81-f7d3-42d6-8f78-27acd7b96b95 tempest-ServerAddressesTestJSON-2064647410 tempest-ServerAddressesTestJSON-2064647410-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:14:52.321355 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-b948de81-f7d3-42d6-8f78-27acd7b96b95 tempest-ServerAddressesTestJSON-2064647410 tempest-ServerAddressesTestJSON-2064647410-project] Getting quotas for project ecf61d694c7f48febd890b942201199a. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 18 22:14:52.325595 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-b948de81-f7d3-42d6-8f78-27acd7b96b95 tempest-ServerAddressesTestJSON-2064647410 tempest-ServerAddressesTestJSON-2064647410-project] Getting quotas for user 3141563506854d988c13410eb97dfac5 and project ecf61d694c7f48febd890b942201199a. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 18 22:14:52.337841 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-b948de81-f7d3-42d6-8f78-27acd7b96b95 tempest-ServerAddressesTestJSON-2064647410 tempest-ServerAddressesTestJSON-2064647410-project] [instance: 75f9f403-8efc-4932-94ea-ce32f58564ce] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120657) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 18 22:14:52.338648 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-b948de81-f7d3-42d6-8f78-27acd7b96b95 tempest-ServerAddressesTestJSON-2064647410 tempest-ServerAddressesTestJSON-2064647410-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:14:52.339077 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-b948de81-f7d3-42d6-8f78-27acd7b96b95 tempest-ServerAddressesTestJSON-2064647410 tempest-ServerAddressesTestJSON-2064647410-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:14:52.339479 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-b948de81-f7d3-42d6-8f78-27acd7b96b95 tempest-ServerAddressesTestJSON-2064647410 tempest-ServerAddressesTestJSON-2064647410-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:14:52.486342 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-b948de81-f7d3-42d6-8f78-27acd7b96b95 tempest-ServerAddressesTestJSON-2064647410 tempest-ServerAddressesTestJSON-2064647410-project] [instance: 75f9f403-8efc-4932-94ea-ce32f58564ce] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='87b0fec9-a948-41f9-a3c0-f0a6a5a8f1e7',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120657) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 18 22:14:52.487902 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-b948de81-f7d3-42d6-8f78-27acd7b96b95 tempest-ServerAddressesTestJSON-2064647410 tempest-ServerAddressesTestJSON-2064647410-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:14:52.488502 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-b948de81-f7d3-42d6-8f78-27acd7b96b95 tempest-ServerAddressesTestJSON-2064647410 tempest-ServerAddressesTestJSON-2064647410-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:14:52.489081 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-b948de81-f7d3-42d6-8f78-27acd7b96b95 tempest-ServerAddressesTestJSON-2064647410 tempest-ServerAddressesTestJSON-2064647410-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:14:52.755430 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-b948de81-f7d3-42d6-8f78-27acd7b96b95 tempest-ServerAddressesTestJSON-2064647410 tempest-ServerAddressesTestJSON-2064647410-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:14:52.756307 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-b948de81-f7d3-42d6-8f78-27acd7b96b95 tempest-ServerAddressesTestJSON-2064647410 tempest-ServerAddressesTestJSON-2064647410-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:14:52.756872 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-b948de81-f7d3-42d6-8f78-27acd7b96b95 tempest-ServerAddressesTestJSON-2064647410 tempest-ServerAddressesTestJSON-2064647410-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:14:52.757344 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-b948de81-f7d3-42d6-8f78-27acd7b96b95 tempest-ServerAddressesTestJSON-2064647410 tempest-ServerAddressesTestJSON-2064647410-project] Acquiring lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 18 22:14:52.757691 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-b948de81-f7d3-42d6-8f78-27acd7b96b95 tempest-ServerAddressesTestJSON-2064647410 tempest-ServerAddressesTestJSON-2064647410-project] Acquired lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 18 22:14:52.759222 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-6c91e171-2910-4648-8d28-2abf97deead9 None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:14:52.759643 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-6c91e171-2910-4648-8d28-2abf97deead9 None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:14:52.764543 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-6c91e171-2910-4648-8d28-2abf97deead9 None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:14:52.764543 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-6c91e171-2910-4648-8d28-2abf97deead9 None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:14:52.764543 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-6c91e171-2910-4648-8d28-2abf97deead9 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:14:52.764543 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-6c91e171-2910-4648-8d28-2abf97deead9 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:14:52.780102 n-d-853704-1 nova-conductor[120657]: ERROR nova.compute.rpcapi [None req-6c91e171-2910-4648-8d28-2abf97deead9 None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 18 22:14:52.780505 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-6c91e171-2910-4648-8d28-2abf97deead9 None None] Releasing lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 18 22:14:52.781539 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server [None req-6c91e171-2910-4648-8d28-2abf97deead9 None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:14:52.781539 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:14:52.781539 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 18 22:14:52.781539 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 18 22:14:52.781539 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 18 22:14:52.781539 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 22:14:52.781539 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 18 22:14:52.781539 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 22:14:52.781539 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:14:52.781539 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 18 22:14:52.781539 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 18 22:14:52.781539 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 18 22:14:52.781539 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 18 22:14:52.781539 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 18 22:14:52.781539 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 18 22:14:52.781539 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 18 22:14:52.781539 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 18 22:14:52.783796 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 18 22:14:52.783796 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 18 22:14:52.783796 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 18 22:14:52.783796 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 18 22:14:52.783796 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 18 22:14:52.783796 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 18 22:14:52.783796 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:14:52.783796 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 22:14:56.093845 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-b2a60790-24a5-447e-881f-08556c445025 tempest-ServersAdminTestJSON-1173706145 tempest-ServersAdminTestJSON-1173706145-project] Took 0.88 seconds to select destinations for 1 instance(s). {{(pid=120658) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 18 22:14:56.121699 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-b2a60790-24a5-447e-881f-08556c445025 tempest-ServersAdminTestJSON-1173706145 tempest-ServersAdminTestJSON-1173706145-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:14:56.122216 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-b2a60790-24a5-447e-881f-08556c445025 tempest-ServersAdminTestJSON-1173706145 tempest-ServersAdminTestJSON-1173706145-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:14:56.122796 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-b2a60790-24a5-447e-881f-08556c445025 tempest-ServersAdminTestJSON-1173706145 tempest-ServersAdminTestJSON-1173706145-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:14:56.439843 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-b2a60790-24a5-447e-881f-08556c445025 tempest-ServersAdminTestJSON-1173706145 tempest-ServersAdminTestJSON-1173706145-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:14:56.440304 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-b2a60790-24a5-447e-881f-08556c445025 tempest-ServersAdminTestJSON-1173706145 tempest-ServersAdminTestJSON-1173706145-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:14:56.440804 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-b2a60790-24a5-447e-881f-08556c445025 tempest-ServersAdminTestJSON-1173706145 tempest-ServersAdminTestJSON-1173706145-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:14:56.441548 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-b2a60790-24a5-447e-881f-08556c445025 tempest-ServersAdminTestJSON-1173706145 tempest-ServersAdminTestJSON-1173706145-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:14:56.441973 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-b2a60790-24a5-447e-881f-08556c445025 tempest-ServersAdminTestJSON-1173706145 tempest-ServersAdminTestJSON-1173706145-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:14:56.442352 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-b2a60790-24a5-447e-881f-08556c445025 tempest-ServersAdminTestJSON-1173706145 tempest-ServersAdminTestJSON-1173706145-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:14:56.463007 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-b2a60790-24a5-447e-881f-08556c445025 tempest-ServersAdminTestJSON-1173706145 tempest-ServersAdminTestJSON-1173706145-project] Getting quotas for project 796c75334d5d4d91a82e04455d65c196. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 18 22:14:56.468869 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-b2a60790-24a5-447e-881f-08556c445025 tempest-ServersAdminTestJSON-1173706145 tempest-ServersAdminTestJSON-1173706145-project] Getting quotas for user 75c9022a7cc84f74aa717cea30f9f42b and project 796c75334d5d4d91a82e04455d65c196. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 18 22:14:56.483089 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-b2a60790-24a5-447e-881f-08556c445025 tempest-ServersAdminTestJSON-1173706145 tempest-ServersAdminTestJSON-1173706145-project] [instance: 80319327-60a1-441e-a3f3-1828cb84753f] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120658) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 18 22:14:56.484472 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-b2a60790-24a5-447e-881f-08556c445025 tempest-ServersAdminTestJSON-1173706145 tempest-ServersAdminTestJSON-1173706145-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:14:56.485092 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-b2a60790-24a5-447e-881f-08556c445025 tempest-ServersAdminTestJSON-1173706145 tempest-ServersAdminTestJSON-1173706145-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:14:56.486194 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-b2a60790-24a5-447e-881f-08556c445025 tempest-ServersAdminTestJSON-1173706145 tempest-ServersAdminTestJSON-1173706145-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:14:56.586341 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-b2a60790-24a5-447e-881f-08556c445025 tempest-ServersAdminTestJSON-1173706145 tempest-ServersAdminTestJSON-1173706145-project] [instance: 80319327-60a1-441e-a3f3-1828cb84753f] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='87b0fec9-a948-41f9-a3c0-f0a6a5a8f1e7',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120658) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 18 22:14:56.587688 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-b2a60790-24a5-447e-881f-08556c445025 tempest-ServersAdminTestJSON-1173706145 tempest-ServersAdminTestJSON-1173706145-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:14:56.588157 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-b2a60790-24a5-447e-881f-08556c445025 tempest-ServersAdminTestJSON-1173706145 tempest-ServersAdminTestJSON-1173706145-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:14:56.588562 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-b2a60790-24a5-447e-881f-08556c445025 tempest-ServersAdminTestJSON-1173706145 tempest-ServersAdminTestJSON-1173706145-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:14:56.837074 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-642d4ca2-8375-4608-bbad-6adc32515401 tempest-ServersAdminTestJSON-1173706145 tempest-ServersAdminTestJSON-1173706145-project] Took 0.52 seconds to select destinations for 1 instance(s). {{(pid=120657) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 18 22:14:56.858050 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-642d4ca2-8375-4608-bbad-6adc32515401 tempest-ServersAdminTestJSON-1173706145 tempest-ServersAdminTestJSON-1173706145-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:14:56.858497 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-642d4ca2-8375-4608-bbad-6adc32515401 tempest-ServersAdminTestJSON-1173706145 tempest-ServersAdminTestJSON-1173706145-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:14:56.863914 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-642d4ca2-8375-4608-bbad-6adc32515401 tempest-ServersAdminTestJSON-1173706145 tempest-ServersAdminTestJSON-1173706145-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:14:56.918180 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-b2a60790-24a5-447e-881f-08556c445025 tempest-ServersAdminTestJSON-1173706145 tempest-ServersAdminTestJSON-1173706145-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:14:56.918633 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-b2a60790-24a5-447e-881f-08556c445025 tempest-ServersAdminTestJSON-1173706145 tempest-ServersAdminTestJSON-1173706145-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:14:56.919036 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-b2a60790-24a5-447e-881f-08556c445025 tempest-ServersAdminTestJSON-1173706145 tempest-ServersAdminTestJSON-1173706145-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:14:56.919516 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-b2a60790-24a5-447e-881f-08556c445025 tempest-ServersAdminTestJSON-1173706145 tempest-ServersAdminTestJSON-1173706145-project] Acquiring lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 18 22:14:56.919902 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-b2a60790-24a5-447e-881f-08556c445025 tempest-ServersAdminTestJSON-1173706145 tempest-ServersAdminTestJSON-1173706145-project] Acquired lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 18 22:14:56.920650 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-16c7af17-cd7c-475d-ae12-1bb391cf1461 None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:14:56.921089 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-16c7af17-cd7c-475d-ae12-1bb391cf1461 None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:14:56.921515 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-16c7af17-cd7c-475d-ae12-1bb391cf1461 None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:14:56.922352 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-16c7af17-cd7c-475d-ae12-1bb391cf1461 None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:14:56.923976 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-16c7af17-cd7c-475d-ae12-1bb391cf1461 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:14:56.927117 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-16c7af17-cd7c-475d-ae12-1bb391cf1461 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:14:56.933089 n-d-853704-1 nova-conductor[120658]: ERROR nova.compute.rpcapi [None req-16c7af17-cd7c-475d-ae12-1bb391cf1461 None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 18 22:14:56.936182 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-16c7af17-cd7c-475d-ae12-1bb391cf1461 None None] Releasing lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 18 22:14:56.939478 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server [None req-16c7af17-cd7c-475d-ae12-1bb391cf1461 None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:14:56.939478 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:14:56.939478 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 18 22:14:56.939478 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 18 22:14:56.939478 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 18 22:14:56.939478 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 22:14:56.939478 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 18 22:14:56.939478 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 22:14:56.939478 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:14:56.939478 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 18 22:14:56.939478 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 18 22:14:56.939478 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 18 22:14:56.939478 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 18 22:14:56.939478 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 18 22:14:56.939478 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 18 22:14:56.939478 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 18 22:14:56.939478 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 18 22:14:56.941463 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 18 22:14:56.941463 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 18 22:14:56.941463 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 18 22:14:56.941463 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 18 22:14:56.941463 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 18 22:14:56.941463 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 18 22:14:56.941463 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:14:56.941463 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 22:14:57.059226 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-642d4ca2-8375-4608-bbad-6adc32515401 tempest-ServersAdminTestJSON-1173706145 tempest-ServersAdminTestJSON-1173706145-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:14:57.059693 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-642d4ca2-8375-4608-bbad-6adc32515401 tempest-ServersAdminTestJSON-1173706145 tempest-ServersAdminTestJSON-1173706145-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:14:57.060242 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-642d4ca2-8375-4608-bbad-6adc32515401 tempest-ServersAdminTestJSON-1173706145 tempest-ServersAdminTestJSON-1173706145-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:14:57.060914 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-642d4ca2-8375-4608-bbad-6adc32515401 tempest-ServersAdminTestJSON-1173706145 tempest-ServersAdminTestJSON-1173706145-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:14:57.061335 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-642d4ca2-8375-4608-bbad-6adc32515401 tempest-ServersAdminTestJSON-1173706145 tempest-ServersAdminTestJSON-1173706145-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:14:57.061728 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-642d4ca2-8375-4608-bbad-6adc32515401 tempest-ServersAdminTestJSON-1173706145 tempest-ServersAdminTestJSON-1173706145-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:14:57.082313 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-642d4ca2-8375-4608-bbad-6adc32515401 tempest-ServersAdminTestJSON-1173706145 tempest-ServersAdminTestJSON-1173706145-project] Getting quotas for project 796c75334d5d4d91a82e04455d65c196. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 18 22:14:57.086923 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-642d4ca2-8375-4608-bbad-6adc32515401 tempest-ServersAdminTestJSON-1173706145 tempest-ServersAdminTestJSON-1173706145-project] Getting quotas for user 75c9022a7cc84f74aa717cea30f9f42b and project 796c75334d5d4d91a82e04455d65c196. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 18 22:14:57.106615 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-642d4ca2-8375-4608-bbad-6adc32515401 tempest-ServersAdminTestJSON-1173706145 tempest-ServersAdminTestJSON-1173706145-project] [instance: 587b0ac9-af53-4483-986b-999f3a2a372a] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120657) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 18 22:14:57.108086 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-642d4ca2-8375-4608-bbad-6adc32515401 tempest-ServersAdminTestJSON-1173706145 tempest-ServersAdminTestJSON-1173706145-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:14:57.108705 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-642d4ca2-8375-4608-bbad-6adc32515401 tempest-ServersAdminTestJSON-1173706145 tempest-ServersAdminTestJSON-1173706145-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:14:57.109275 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-642d4ca2-8375-4608-bbad-6adc32515401 tempest-ServersAdminTestJSON-1173706145 tempest-ServersAdminTestJSON-1173706145-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:14:57.214709 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-642d4ca2-8375-4608-bbad-6adc32515401 tempest-ServersAdminTestJSON-1173706145 tempest-ServersAdminTestJSON-1173706145-project] [instance: 587b0ac9-af53-4483-986b-999f3a2a372a] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='87b0fec9-a948-41f9-a3c0-f0a6a5a8f1e7',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120657) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 18 22:14:57.215982 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-642d4ca2-8375-4608-bbad-6adc32515401 tempest-ServersAdminTestJSON-1173706145 tempest-ServersAdminTestJSON-1173706145-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:14:57.216624 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-642d4ca2-8375-4608-bbad-6adc32515401 tempest-ServersAdminTestJSON-1173706145 tempest-ServersAdminTestJSON-1173706145-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:14:57.217239 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-642d4ca2-8375-4608-bbad-6adc32515401 tempest-ServersAdminTestJSON-1173706145 tempest-ServersAdminTestJSON-1173706145-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:14:57.513459 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-642d4ca2-8375-4608-bbad-6adc32515401 tempest-ServersAdminTestJSON-1173706145 tempest-ServersAdminTestJSON-1173706145-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:14:57.516181 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-642d4ca2-8375-4608-bbad-6adc32515401 tempest-ServersAdminTestJSON-1173706145 tempest-ServersAdminTestJSON-1173706145-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:14:57.516181 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-642d4ca2-8375-4608-bbad-6adc32515401 tempest-ServersAdminTestJSON-1173706145 tempest-ServersAdminTestJSON-1173706145-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:14:57.516181 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-642d4ca2-8375-4608-bbad-6adc32515401 tempest-ServersAdminTestJSON-1173706145 tempest-ServersAdminTestJSON-1173706145-project] Acquiring lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 18 22:14:57.516181 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-642d4ca2-8375-4608-bbad-6adc32515401 tempest-ServersAdminTestJSON-1173706145 tempest-ServersAdminTestJSON-1173706145-project] Acquired lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 18 22:14:57.517124 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-20c8dd25-e6c6-43e0-b0a4-cc3204420bdf None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:14:57.518384 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-20c8dd25-e6c6-43e0-b0a4-cc3204420bdf None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:14:57.519341 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-20c8dd25-e6c6-43e0-b0a4-cc3204420bdf None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:14:57.520407 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-20c8dd25-e6c6-43e0-b0a4-cc3204420bdf None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:14:57.521057 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-20c8dd25-e6c6-43e0-b0a4-cc3204420bdf None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:14:57.521742 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-20c8dd25-e6c6-43e0-b0a4-cc3204420bdf None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:14:57.534488 n-d-853704-1 nova-conductor[120657]: ERROR nova.compute.rpcapi [None req-20c8dd25-e6c6-43e0-b0a4-cc3204420bdf None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 18 22:14:57.535354 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-20c8dd25-e6c6-43e0-b0a4-cc3204420bdf None None] Releasing lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 18 22:14:57.536738 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server [None req-20c8dd25-e6c6-43e0-b0a4-cc3204420bdf None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:14:57.536738 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:14:57.536738 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 18 22:14:57.536738 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 18 22:14:57.536738 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 18 22:14:57.536738 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 22:14:57.536738 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 18 22:14:57.536738 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 22:14:57.536738 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:14:57.536738 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 18 22:14:57.536738 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 18 22:14:57.536738 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 18 22:14:57.536738 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 18 22:14:57.536738 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 18 22:14:57.536738 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 18 22:14:57.536738 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 18 22:14:57.536738 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 18 22:14:57.538119 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 18 22:14:57.538119 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 18 22:14:57.538119 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 18 22:14:57.538119 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 18 22:14:57.538119 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 18 22:14:57.538119 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 18 22:14:57.538119 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:14:57.538119 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 22:15:04.860874 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-b9cb9e98-574e-4acb-8a81-976edac40731 tempest-ServersTestJSON-1241713830 tempest-ServersTestJSON-1241713830-project] Took 0.54 seconds to select destinations for 1 instance(s). {{(pid=120658) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 18 22:15:04.881421 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-b9cb9e98-574e-4acb-8a81-976edac40731 tempest-ServersTestJSON-1241713830 tempest-ServersTestJSON-1241713830-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:15:04.881877 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-b9cb9e98-574e-4acb-8a81-976edac40731 tempest-ServersTestJSON-1241713830 tempest-ServersTestJSON-1241713830-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:15:04.882284 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-b9cb9e98-574e-4acb-8a81-976edac40731 tempest-ServersTestJSON-1241713830 tempest-ServersTestJSON-1241713830-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:15:05.091239 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-b9cb9e98-574e-4acb-8a81-976edac40731 tempest-ServersTestJSON-1241713830 tempest-ServersTestJSON-1241713830-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:15:05.091716 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-b9cb9e98-574e-4acb-8a81-976edac40731 tempest-ServersTestJSON-1241713830 tempest-ServersTestJSON-1241713830-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:15:05.092131 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-b9cb9e98-574e-4acb-8a81-976edac40731 tempest-ServersTestJSON-1241713830 tempest-ServersTestJSON-1241713830-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:15:05.092807 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-b9cb9e98-574e-4acb-8a81-976edac40731 tempest-ServersTestJSON-1241713830 tempest-ServersTestJSON-1241713830-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:15:05.093235 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-b9cb9e98-574e-4acb-8a81-976edac40731 tempest-ServersTestJSON-1241713830 tempest-ServersTestJSON-1241713830-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:15:05.093631 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-b9cb9e98-574e-4acb-8a81-976edac40731 tempest-ServersTestJSON-1241713830 tempest-ServersTestJSON-1241713830-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:15:05.109396 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-b9cb9e98-574e-4acb-8a81-976edac40731 tempest-ServersTestJSON-1241713830 tempest-ServersTestJSON-1241713830-project] Getting quotas for project 6304d529bdcb4eb4b60976d5ccb12282. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 18 22:15:05.113922 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-b9cb9e98-574e-4acb-8a81-976edac40731 tempest-ServersTestJSON-1241713830 tempest-ServersTestJSON-1241713830-project] Getting quotas for user 663f23df62b24a28a4e08141b63743b2 and project 6304d529bdcb4eb4b60976d5ccb12282. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 18 22:15:05.126138 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-b9cb9e98-574e-4acb-8a81-976edac40731 tempest-ServersTestJSON-1241713830 tempest-ServersTestJSON-1241713830-project] [instance: 291a32d9-89db-4f18-9993-d5da617daba1] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120658) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 18 22:15:05.126976 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-b9cb9e98-574e-4acb-8a81-976edac40731 tempest-ServersTestJSON-1241713830 tempest-ServersTestJSON-1241713830-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:15:05.127497 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-b9cb9e98-574e-4acb-8a81-976edac40731 tempest-ServersTestJSON-1241713830 tempest-ServersTestJSON-1241713830-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:15:05.127927 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-b9cb9e98-574e-4acb-8a81-976edac40731 tempest-ServersTestJSON-1241713830 tempest-ServersTestJSON-1241713830-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:15:05.214118 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-b9cb9e98-574e-4acb-8a81-976edac40731 tempest-ServersTestJSON-1241713830 tempest-ServersTestJSON-1241713830-project] [instance: 291a32d9-89db-4f18-9993-d5da617daba1] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='87b0fec9-a948-41f9-a3c0-f0a6a5a8f1e7',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120658) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 18 22:15:05.215284 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-b9cb9e98-574e-4acb-8a81-976edac40731 tempest-ServersTestJSON-1241713830 tempest-ServersTestJSON-1241713830-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:15:05.215756 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-b9cb9e98-574e-4acb-8a81-976edac40731 tempest-ServersTestJSON-1241713830 tempest-ServersTestJSON-1241713830-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:15:05.216164 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-b9cb9e98-574e-4acb-8a81-976edac40731 tempest-ServersTestJSON-1241713830 tempest-ServersTestJSON-1241713830-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:15:05.477812 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-b9cb9e98-574e-4acb-8a81-976edac40731 tempest-ServersTestJSON-1241713830 tempest-ServersTestJSON-1241713830-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:15:05.478205 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-b9cb9e98-574e-4acb-8a81-976edac40731 tempest-ServersTestJSON-1241713830 tempest-ServersTestJSON-1241713830-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:15:05.478605 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-b9cb9e98-574e-4acb-8a81-976edac40731 tempest-ServersTestJSON-1241713830 tempest-ServersTestJSON-1241713830-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:15:05.479065 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-b9cb9e98-574e-4acb-8a81-976edac40731 tempest-ServersTestJSON-1241713830 tempest-ServersTestJSON-1241713830-project] Acquiring lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 18 22:15:05.479448 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-b9cb9e98-574e-4acb-8a81-976edac40731 tempest-ServersTestJSON-1241713830 tempest-ServersTestJSON-1241713830-project] Acquired lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 18 22:15:05.480228 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-600b9c5d-e076-4714-b83b-0bf9a2515ab7 None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:15:05.480672 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-600b9c5d-e076-4714-b83b-0bf9a2515ab7 None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:15:05.481085 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-600b9c5d-e076-4714-b83b-0bf9a2515ab7 None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:15:05.481736 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-600b9c5d-e076-4714-b83b-0bf9a2515ab7 None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:15:05.482149 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-600b9c5d-e076-4714-b83b-0bf9a2515ab7 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:15:05.482538 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-600b9c5d-e076-4714-b83b-0bf9a2515ab7 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:15:05.493793 n-d-853704-1 nova-conductor[120658]: ERROR nova.compute.rpcapi [None req-600b9c5d-e076-4714-b83b-0bf9a2515ab7 None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 18 22:15:05.494193 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-600b9c5d-e076-4714-b83b-0bf9a2515ab7 None None] Releasing lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 18 22:15:05.495103 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server [None req-600b9c5d-e076-4714-b83b-0bf9a2515ab7 None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:15:05.495103 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:15:05.495103 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 18 22:15:05.495103 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 18 22:15:05.495103 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 18 22:15:05.495103 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 22:15:05.495103 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 18 22:15:05.495103 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 22:15:05.495103 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:15:05.495103 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 18 22:15:05.495103 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 18 22:15:05.495103 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 18 22:15:05.495103 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 18 22:15:05.495103 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 18 22:15:05.495103 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 18 22:15:05.495103 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 18 22:15:05.495103 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 18 22:15:05.496640 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 18 22:15:05.496640 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 18 22:15:05.496640 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 18 22:15:05.496640 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 18 22:15:05.496640 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 18 22:15:05.496640 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 18 22:15:05.496640 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:15:05.496640 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 22:15:07.446344 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=32,nova_api:UPDATE=2,nova_api:DELETE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:15:07.447069 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=40,nova_api:DELETE=2,nova_api:UPDATE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:15:07.534700 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=12 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:15:07.535292 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell1:SELECT=33,nova_cell1:SAVEPOINT=2,nova_cell1:INSERT=53,nova_cell1:RELEASE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:15:12.984590 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=26,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:15:15.387805 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=33,nova_api:UPDATE=2,nova_api:DELETE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:15:15.388577 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=39,nova_api:UPDATE=1,nova_api:DELETE=2 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:15:15.488985 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=12 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:15:15.492703 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell1:SELECT=33,nova_cell1:INSERT=54,nova_cell1:SAVEPOINT=2,nova_cell1:RELEASE=2 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:15:23.025385 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=27,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:15:52.987181 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:16:03.029673 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:16:22.988744 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=5,nova_cell0:UPDATE=3 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:16:43.030023 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:17:02.989405 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:17:23.030070 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:17:42.988650 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:17:52.933096 n-d-853704-1 nova-conductor[120657]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 18 22:18:03.036943 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=7,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:18:12.999491 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=5,nova_cell0:UPDATE=3 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:18:21.228224 n-d-853704-1 nova-conductor[120658]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 18 22:18:43.038419 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:18:50.550932 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-1a65d644-c64a-45dc-a192-649e112f3586 tempest-ServersAdmin275Test-1284551606 tempest-ServersAdmin275Test-1284551606-project] Took 0.66 seconds to select destinations for 1 instance(s). {{(pid=120657) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 18 22:18:50.575519 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-1a65d644-c64a-45dc-a192-649e112f3586 tempest-ServersAdmin275Test-1284551606 tempest-ServersAdmin275Test-1284551606-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:18:50.576018 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-1a65d644-c64a-45dc-a192-649e112f3586 tempest-ServersAdmin275Test-1284551606 tempest-ServersAdmin275Test-1284551606-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:18:50.576415 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-1a65d644-c64a-45dc-a192-649e112f3586 tempest-ServersAdmin275Test-1284551606 tempest-ServersAdmin275Test-1284551606-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:18:50.762085 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:18:50.765307 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-1a65d644-c64a-45dc-a192-649e112f3586 tempest-ServersAdmin275Test-1284551606 tempest-ServersAdmin275Test-1284551606-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:18:50.765755 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-1a65d644-c64a-45dc-a192-649e112f3586 tempest-ServersAdmin275Test-1284551606 tempest-ServersAdmin275Test-1284551606-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:18:50.766126 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-1a65d644-c64a-45dc-a192-649e112f3586 tempest-ServersAdmin275Test-1284551606 tempest-ServersAdmin275Test-1284551606-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:18:50.767539 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-1a65d644-c64a-45dc-a192-649e112f3586 tempest-ServersAdmin275Test-1284551606 tempest-ServersAdmin275Test-1284551606-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:18:50.768165 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-1a65d644-c64a-45dc-a192-649e112f3586 tempest-ServersAdmin275Test-1284551606 tempest-ServersAdmin275Test-1284551606-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:18:50.768594 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-1a65d644-c64a-45dc-a192-649e112f3586 tempest-ServersAdmin275Test-1284551606 tempest-ServersAdmin275Test-1284551606-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:18:50.786660 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-1a65d644-c64a-45dc-a192-649e112f3586 tempest-ServersAdmin275Test-1284551606 tempest-ServersAdmin275Test-1284551606-project] Getting quotas for project c9d0989473c14f1cbcbebc530a6a9d43. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 18 22:18:50.790911 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-1a65d644-c64a-45dc-a192-649e112f3586 tempest-ServersAdmin275Test-1284551606 tempest-ServersAdmin275Test-1284551606-project] Getting quotas for user ac7b0b79798c4dc7a9abf0d14b499072 and project c9d0989473c14f1cbcbebc530a6a9d43. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 18 22:18:50.810439 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-1a65d644-c64a-45dc-a192-649e112f3586 tempest-ServersAdmin275Test-1284551606 tempest-ServersAdmin275Test-1284551606-project] [instance: 74a91029-09f8-4107-ae08-a3ad12794953] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120657) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 18 22:18:50.814300 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-1a65d644-c64a-45dc-a192-649e112f3586 tempest-ServersAdmin275Test-1284551606 tempest-ServersAdmin275Test-1284551606-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:18:50.814300 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-1a65d644-c64a-45dc-a192-649e112f3586 tempest-ServersAdmin275Test-1284551606 tempest-ServersAdmin275Test-1284551606-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:18:50.814300 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-1a65d644-c64a-45dc-a192-649e112f3586 tempest-ServersAdmin275Test-1284551606 tempest-ServersAdmin275Test-1284551606-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:18:50.890137 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-1a65d644-c64a-45dc-a192-649e112f3586 tempest-ServersAdmin275Test-1284551606 tempest-ServersAdmin275Test-1284551606-project] [instance: 74a91029-09f8-4107-ae08-a3ad12794953] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='87b0fec9-a948-41f9-a3c0-f0a6a5a8f1e7',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120657) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 18 22:18:50.891448 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-1a65d644-c64a-45dc-a192-649e112f3586 tempest-ServersAdmin275Test-1284551606 tempest-ServersAdmin275Test-1284551606-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:18:50.891954 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-1a65d644-c64a-45dc-a192-649e112f3586 tempest-ServersAdmin275Test-1284551606 tempest-ServersAdmin275Test-1284551606-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:18:50.892355 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-1a65d644-c64a-45dc-a192-649e112f3586 tempest-ServersAdmin275Test-1284551606 tempest-ServersAdmin275Test-1284551606-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:18:51.127956 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-1a65d644-c64a-45dc-a192-649e112f3586 tempest-ServersAdmin275Test-1284551606 tempest-ServersAdmin275Test-1284551606-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:18:51.128407 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-1a65d644-c64a-45dc-a192-649e112f3586 tempest-ServersAdmin275Test-1284551606 tempest-ServersAdmin275Test-1284551606-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:18:51.128807 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-1a65d644-c64a-45dc-a192-649e112f3586 tempest-ServersAdmin275Test-1284551606 tempest-ServersAdmin275Test-1284551606-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:18:51.129292 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-1a65d644-c64a-45dc-a192-649e112f3586 tempest-ServersAdmin275Test-1284551606 tempest-ServersAdmin275Test-1284551606-project] Acquiring lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 18 22:18:51.129646 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-1a65d644-c64a-45dc-a192-649e112f3586 tempest-ServersAdmin275Test-1284551606 tempest-ServersAdmin275Test-1284551606-project] Acquired lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 18 22:18:51.130398 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-14d5c764-c231-4ef9-becc-3e1a2c4cb7aa None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:18:51.130813 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-14d5c764-c231-4ef9-becc-3e1a2c4cb7aa None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:18:51.131198 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-14d5c764-c231-4ef9-becc-3e1a2c4cb7aa None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:18:51.131957 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-14d5c764-c231-4ef9-becc-3e1a2c4cb7aa None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:18:51.132362 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-14d5c764-c231-4ef9-becc-3e1a2c4cb7aa None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:18:51.132745 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-14d5c764-c231-4ef9-becc-3e1a2c4cb7aa None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:18:51.140681 n-d-853704-1 nova-conductor[120657]: ERROR nova.compute.rpcapi [None req-14d5c764-c231-4ef9-becc-3e1a2c4cb7aa None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 18 22:18:51.141069 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-14d5c764-c231-4ef9-becc-3e1a2c4cb7aa None None] Releasing lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 18 22:18:51.142354 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server [None req-14d5c764-c231-4ef9-becc-3e1a2c4cb7aa None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:18:51.142354 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:18:51.142354 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 18 22:18:51.142354 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 18 22:18:51.142354 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 18 22:18:51.142354 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 22:18:51.142354 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 18 22:18:51.142354 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 22:18:51.142354 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:18:51.142354 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 18 22:18:51.142354 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 18 22:18:51.142354 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 18 22:18:51.142354 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 18 22:18:51.142354 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 18 22:18:51.142354 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 18 22:18:51.142354 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 18 22:18:51.142354 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 18 22:18:51.143478 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 18 22:18:51.143478 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 18 22:18:51.143478 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 18 22:18:51.143478 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 18 22:18:51.143478 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 18 22:18:51.143478 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 18 22:18:51.143478 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:18:51.143478 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 22:19:01.059714 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=16,nova_api:UPDATE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:19:01.060584 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=8,nova_api:DELETE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:19:01.138640 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:19:01.140093 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell1:SELECT=11,nova_cell1:SAVEPOINT=1,nova_cell1:INSERT=18,nova_cell1:RELEASE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:19:03.042360 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=3,nova_cell0:UPDATE=2 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:19:23.002248 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=11,nova_cell0:UPDATE=3 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:19:31.412893 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-d13e8e8c-9631-4297-a3fc-e7230d622888 tempest-ImagesTestJSON-1761366649 tempest-ImagesTestJSON-1761366649-project] Took 0.38 seconds to select destinations for 1 instance(s). {{(pid=120658) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 18 22:19:31.433687 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-d13e8e8c-9631-4297-a3fc-e7230d622888 tempest-ImagesTestJSON-1761366649 tempest-ImagesTestJSON-1761366649-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:19:31.434140 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-d13e8e8c-9631-4297-a3fc-e7230d622888 tempest-ImagesTestJSON-1761366649 tempest-ImagesTestJSON-1761366649-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:19:31.434539 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-d13e8e8c-9631-4297-a3fc-e7230d622888 tempest-ImagesTestJSON-1761366649 tempest-ImagesTestJSON-1761366649-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:19:31.530591 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-d13e8e8c-9631-4297-a3fc-e7230d622888 tempest-ImagesTestJSON-1761366649 tempest-ImagesTestJSON-1761366649-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:19:31.531034 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-d13e8e8c-9631-4297-a3fc-e7230d622888 tempest-ImagesTestJSON-1761366649 tempest-ImagesTestJSON-1761366649-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:19:31.531431 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-d13e8e8c-9631-4297-a3fc-e7230d622888 tempest-ImagesTestJSON-1761366649 tempest-ImagesTestJSON-1761366649-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:19:31.532122 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-d13e8e8c-9631-4297-a3fc-e7230d622888 tempest-ImagesTestJSON-1761366649 tempest-ImagesTestJSON-1761366649-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:19:31.532539 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-d13e8e8c-9631-4297-a3fc-e7230d622888 tempest-ImagesTestJSON-1761366649 tempest-ImagesTestJSON-1761366649-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:19:31.532928 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-d13e8e8c-9631-4297-a3fc-e7230d622888 tempest-ImagesTestJSON-1761366649 tempest-ImagesTestJSON-1761366649-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:19:31.549920 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-d13e8e8c-9631-4297-a3fc-e7230d622888 tempest-ImagesTestJSON-1761366649 tempest-ImagesTestJSON-1761366649-project] Getting quotas for project 677821f62eb848bcaf47c31a724218df. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 18 22:19:31.554398 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-d13e8e8c-9631-4297-a3fc-e7230d622888 tempest-ImagesTestJSON-1761366649 tempest-ImagesTestJSON-1761366649-project] Getting quotas for user 2b6fddb708684c6a8d541e617aa2fc48 and project 677821f62eb848bcaf47c31a724218df. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 18 22:19:31.566240 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-d13e8e8c-9631-4297-a3fc-e7230d622888 tempest-ImagesTestJSON-1761366649 tempest-ImagesTestJSON-1761366649-project] [instance: 8acee217-3b2d-4cbf-87e0-57bcbd37fa8f] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120658) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 18 22:19:31.567019 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-d13e8e8c-9631-4297-a3fc-e7230d622888 tempest-ImagesTestJSON-1761366649 tempest-ImagesTestJSON-1761366649-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:19:31.567453 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-d13e8e8c-9631-4297-a3fc-e7230d622888 tempest-ImagesTestJSON-1761366649 tempest-ImagesTestJSON-1761366649-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:19:31.567880 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-d13e8e8c-9631-4297-a3fc-e7230d622888 tempest-ImagesTestJSON-1761366649 tempest-ImagesTestJSON-1761366649-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:19:31.616538 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-d13e8e8c-9631-4297-a3fc-e7230d622888 tempest-ImagesTestJSON-1761366649 tempest-ImagesTestJSON-1761366649-project] [instance: 8acee217-3b2d-4cbf-87e0-57bcbd37fa8f] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='87b0fec9-a948-41f9-a3c0-f0a6a5a8f1e7',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120658) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 18 22:19:31.618481 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-d13e8e8c-9631-4297-a3fc-e7230d622888 tempest-ImagesTestJSON-1761366649 tempest-ImagesTestJSON-1761366649-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:19:31.618991 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-d13e8e8c-9631-4297-a3fc-e7230d622888 tempest-ImagesTestJSON-1761366649 tempest-ImagesTestJSON-1761366649-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:19:31.619458 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-d13e8e8c-9631-4297-a3fc-e7230d622888 tempest-ImagesTestJSON-1761366649 tempest-ImagesTestJSON-1761366649-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:19:31.874304 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-d13e8e8c-9631-4297-a3fc-e7230d622888 tempest-ImagesTestJSON-1761366649 tempest-ImagesTestJSON-1761366649-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:19:31.874666 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-d13e8e8c-9631-4297-a3fc-e7230d622888 tempest-ImagesTestJSON-1761366649 tempest-ImagesTestJSON-1761366649-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:19:31.875058 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-d13e8e8c-9631-4297-a3fc-e7230d622888 tempest-ImagesTestJSON-1761366649 tempest-ImagesTestJSON-1761366649-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:19:31.875516 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-d13e8e8c-9631-4297-a3fc-e7230d622888 tempest-ImagesTestJSON-1761366649 tempest-ImagesTestJSON-1761366649-project] Acquiring lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 18 22:19:31.875899 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-d13e8e8c-9631-4297-a3fc-e7230d622888 tempest-ImagesTestJSON-1761366649 tempest-ImagesTestJSON-1761366649-project] Acquired lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 18 22:19:31.876627 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-a16d8e4d-a658-4502-b236-23a6641d6774 None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:19:31.877038 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-a16d8e4d-a658-4502-b236-23a6641d6774 None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:19:31.877419 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-a16d8e4d-a658-4502-b236-23a6641d6774 None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:19:31.878053 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-a16d8e4d-a658-4502-b236-23a6641d6774 None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:19:31.878456 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-a16d8e4d-a658-4502-b236-23a6641d6774 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:19:31.878832 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-a16d8e4d-a658-4502-b236-23a6641d6774 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:19:31.887442 n-d-853704-1 nova-conductor[120658]: ERROR nova.compute.rpcapi [None req-a16d8e4d-a658-4502-b236-23a6641d6774 None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 18 22:19:31.887921 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-a16d8e4d-a658-4502-b236-23a6641d6774 None None] Releasing lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 18 22:19:31.888826 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server [None req-a16d8e4d-a658-4502-b236-23a6641d6774 None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:19:31.888826 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:19:31.888826 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 18 22:19:31.888826 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 18 22:19:31.888826 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 18 22:19:31.888826 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 22:19:31.888826 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 18 22:19:31.888826 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 22:19:31.888826 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:19:31.888826 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 18 22:19:31.888826 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 18 22:19:31.888826 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 18 22:19:31.888826 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 18 22:19:31.888826 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 18 22:19:31.888826 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 18 22:19:31.888826 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 18 22:19:31.888826 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 18 22:19:31.889994 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 18 22:19:31.889994 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 18 22:19:31.889994 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 18 22:19:31.889994 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 18 22:19:31.889994 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 18 22:19:31.889994 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 18 22:19:31.889994 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:19:31.889994 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 22:19:41.796020 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=11,nova_api:UPDATE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:19:41.796803 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=13,nova_api:DELETE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:19:41.884551 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:19:41.885214 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell1:SELECT=11,nova_cell1:INSERT=17 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:19:43.010496 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=4,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:19:43.051009 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=14,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:20:13.018955 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=6,nova_cell0:UPDATE=3 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:20:17.597674 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-77f10fc9-c168-4acb-bf31-49430509bb63 tempest-MigrationsAdminTest-1637837745 tempest-MigrationsAdminTest-1637837745-project] Took 0.41 seconds to select destinations for 1 instance(s). {{(pid=120657) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 18 22:20:17.618755 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-77f10fc9-c168-4acb-bf31-49430509bb63 tempest-MigrationsAdminTest-1637837745 tempest-MigrationsAdminTest-1637837745-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:20:17.619218 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-77f10fc9-c168-4acb-bf31-49430509bb63 tempest-MigrationsAdminTest-1637837745 tempest-MigrationsAdminTest-1637837745-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:20:17.619606 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-77f10fc9-c168-4acb-bf31-49430509bb63 tempest-MigrationsAdminTest-1637837745 tempest-MigrationsAdminTest-1637837745-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:20:17.780650 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-77f10fc9-c168-4acb-bf31-49430509bb63 tempest-MigrationsAdminTest-1637837745 tempest-MigrationsAdminTest-1637837745-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:20:17.781021 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-77f10fc9-c168-4acb-bf31-49430509bb63 tempest-MigrationsAdminTest-1637837745 tempest-MigrationsAdminTest-1637837745-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:20:17.781418 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-77f10fc9-c168-4acb-bf31-49430509bb63 tempest-MigrationsAdminTest-1637837745 tempest-MigrationsAdminTest-1637837745-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:20:17.782081 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-77f10fc9-c168-4acb-bf31-49430509bb63 tempest-MigrationsAdminTest-1637837745 tempest-MigrationsAdminTest-1637837745-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:20:17.782500 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-77f10fc9-c168-4acb-bf31-49430509bb63 tempest-MigrationsAdminTest-1637837745 tempest-MigrationsAdminTest-1637837745-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:20:17.782886 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-77f10fc9-c168-4acb-bf31-49430509bb63 tempest-MigrationsAdminTest-1637837745 tempest-MigrationsAdminTest-1637837745-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:20:17.799288 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-77f10fc9-c168-4acb-bf31-49430509bb63 tempest-MigrationsAdminTest-1637837745 tempest-MigrationsAdminTest-1637837745-project] Getting quotas for project 55cd873dadd6496dbebea5c3290775eb. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 18 22:20:17.803462 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-77f10fc9-c168-4acb-bf31-49430509bb63 tempest-MigrationsAdminTest-1637837745 tempest-MigrationsAdminTest-1637837745-project] Getting quotas for user caa4813c18704375b16dedceb822364f and project 55cd873dadd6496dbebea5c3290775eb. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 18 22:20:17.815082 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-77f10fc9-c168-4acb-bf31-49430509bb63 tempest-MigrationsAdminTest-1637837745 tempest-MigrationsAdminTest-1637837745-project] [instance: 4c5f9fe0-5bf9-48e6-8070-e96721caf026] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120657) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 18 22:20:17.815902 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-77f10fc9-c168-4acb-bf31-49430509bb63 tempest-MigrationsAdminTest-1637837745 tempest-MigrationsAdminTest-1637837745-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:20:17.816349 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-77f10fc9-c168-4acb-bf31-49430509bb63 tempest-MigrationsAdminTest-1637837745 tempest-MigrationsAdminTest-1637837745-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:20:17.816747 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-77f10fc9-c168-4acb-bf31-49430509bb63 tempest-MigrationsAdminTest-1637837745 tempest-MigrationsAdminTest-1637837745-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:20:17.969565 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-77f10fc9-c168-4acb-bf31-49430509bb63 tempest-MigrationsAdminTest-1637837745 tempest-MigrationsAdminTest-1637837745-project] [instance: 4c5f9fe0-5bf9-48e6-8070-e96721caf026] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='87b0fec9-a948-41f9-a3c0-f0a6a5a8f1e7',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120657) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 18 22:20:17.970816 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-77f10fc9-c168-4acb-bf31-49430509bb63 tempest-MigrationsAdminTest-1637837745 tempest-MigrationsAdminTest-1637837745-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:20:17.971267 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-77f10fc9-c168-4acb-bf31-49430509bb63 tempest-MigrationsAdminTest-1637837745 tempest-MigrationsAdminTest-1637837745-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:20:17.971753 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-77f10fc9-c168-4acb-bf31-49430509bb63 tempest-MigrationsAdminTest-1637837745 tempest-MigrationsAdminTest-1637837745-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:20:18.347259 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-77f10fc9-c168-4acb-bf31-49430509bb63 tempest-MigrationsAdminTest-1637837745 tempest-MigrationsAdminTest-1637837745-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:20:18.347730 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-77f10fc9-c168-4acb-bf31-49430509bb63 tempest-MigrationsAdminTest-1637837745 tempest-MigrationsAdminTest-1637837745-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:20:18.348130 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-77f10fc9-c168-4acb-bf31-49430509bb63 tempest-MigrationsAdminTest-1637837745 tempest-MigrationsAdminTest-1637837745-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:20:18.348610 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-77f10fc9-c168-4acb-bf31-49430509bb63 tempest-MigrationsAdminTest-1637837745 tempest-MigrationsAdminTest-1637837745-project] Acquiring lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 18 22:20:18.349137 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-77f10fc9-c168-4acb-bf31-49430509bb63 tempest-MigrationsAdminTest-1637837745 tempest-MigrationsAdminTest-1637837745-project] Acquired lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 18 22:20:18.349869 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-acc17675-2913-4a8e-9558-b15036e443e2 None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:20:18.350282 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-acc17675-2913-4a8e-9558-b15036e443e2 None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:20:18.350665 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-acc17675-2913-4a8e-9558-b15036e443e2 None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:20:18.351313 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-acc17675-2913-4a8e-9558-b15036e443e2 None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:20:18.351741 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-acc17675-2913-4a8e-9558-b15036e443e2 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:20:18.352681 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-acc17675-2913-4a8e-9558-b15036e443e2 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:20:18.361588 n-d-853704-1 nova-conductor[120657]: ERROR nova.compute.rpcapi [None req-acc17675-2913-4a8e-9558-b15036e443e2 None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 18 22:20:18.361927 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-acc17675-2913-4a8e-9558-b15036e443e2 None None] Releasing lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 18 22:20:18.362839 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server [None req-acc17675-2913-4a8e-9558-b15036e443e2 None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:20:18.362839 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:20:18.362839 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 18 22:20:18.362839 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 18 22:20:18.362839 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 18 22:20:18.362839 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 22:20:18.362839 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 18 22:20:18.362839 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 22:20:18.362839 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:20:18.362839 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 18 22:20:18.362839 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 18 22:20:18.362839 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 18 22:20:18.362839 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 18 22:20:18.362839 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 18 22:20:18.362839 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 18 22:20:18.362839 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 18 22:20:18.362839 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 18 22:20:18.368504 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 18 22:20:18.368504 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 18 22:20:18.368504 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 18 22:20:18.368504 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 18 22:20:18.368504 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 18 22:20:18.368504 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 18 22:20:18.368504 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:20:18.368504 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 22:20:23.059715 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:20:28.262883 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=14,nova_api:UPDATE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:20:28.263645 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=10,nova_api:DELETE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:20:28.360321 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:20:28.362781 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell1:SELECT=11,nova_cell1:INSERT=17 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:20:43.026127 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=12,nova_cell0:UPDATE=3 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:20:43.066091 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=3,nova_cell0:UPDATE=2 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:20:53.479780 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-5fd4340e-1c2a-4091-8cc5-54daee2c4c82 tempest-SecurityGroupsTestJSON-433475947 tempest-SecurityGroupsTestJSON-433475947-project] Took 0.49 seconds to select destinations for 1 instance(s). {{(pid=120658) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 18 22:20:53.508577 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-5fd4340e-1c2a-4091-8cc5-54daee2c4c82 tempest-SecurityGroupsTestJSON-433475947 tempest-SecurityGroupsTestJSON-433475947-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:20:53.508746 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-5fd4340e-1c2a-4091-8cc5-54daee2c4c82 tempest-SecurityGroupsTestJSON-433475947 tempest-SecurityGroupsTestJSON-433475947-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:20:53.509074 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-5fd4340e-1c2a-4091-8cc5-54daee2c4c82 tempest-SecurityGroupsTestJSON-433475947 tempest-SecurityGroupsTestJSON-433475947-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:20:53.630284 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-5fd4340e-1c2a-4091-8cc5-54daee2c4c82 tempest-SecurityGroupsTestJSON-433475947 tempest-SecurityGroupsTestJSON-433475947-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:20:53.631461 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-5fd4340e-1c2a-4091-8cc5-54daee2c4c82 tempest-SecurityGroupsTestJSON-433475947 tempest-SecurityGroupsTestJSON-433475947-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:20:53.632053 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-5fd4340e-1c2a-4091-8cc5-54daee2c4c82 tempest-SecurityGroupsTestJSON-433475947 tempest-SecurityGroupsTestJSON-433475947-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:20:53.632899 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-5fd4340e-1c2a-4091-8cc5-54daee2c4c82 tempest-SecurityGroupsTestJSON-433475947 tempest-SecurityGroupsTestJSON-433475947-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:20:53.633469 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-5fd4340e-1c2a-4091-8cc5-54daee2c4c82 tempest-SecurityGroupsTestJSON-433475947 tempest-SecurityGroupsTestJSON-433475947-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:20:53.634024 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-5fd4340e-1c2a-4091-8cc5-54daee2c4c82 tempest-SecurityGroupsTestJSON-433475947 tempest-SecurityGroupsTestJSON-433475947-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:20:53.659446 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-5fd4340e-1c2a-4091-8cc5-54daee2c4c82 tempest-SecurityGroupsTestJSON-433475947 tempest-SecurityGroupsTestJSON-433475947-project] Getting quotas for project f1a2a950c02b41c08f58cda4453a2865. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 18 22:20:53.664731 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-5fd4340e-1c2a-4091-8cc5-54daee2c4c82 tempest-SecurityGroupsTestJSON-433475947 tempest-SecurityGroupsTestJSON-433475947-project] Getting quotas for user 8e8024c8bb464e189bc0ac5069891bb9 and project f1a2a950c02b41c08f58cda4453a2865. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 18 22:20:53.677611 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-5fd4340e-1c2a-4091-8cc5-54daee2c4c82 tempest-SecurityGroupsTestJSON-433475947 tempest-SecurityGroupsTestJSON-433475947-project] [instance: 8cded6c0-70d3-4172-8c68-d7632a3f503f] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120658) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 18 22:20:53.678640 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-5fd4340e-1c2a-4091-8cc5-54daee2c4c82 tempest-SecurityGroupsTestJSON-433475947 tempest-SecurityGroupsTestJSON-433475947-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:20:53.681573 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-5fd4340e-1c2a-4091-8cc5-54daee2c4c82 tempest-SecurityGroupsTestJSON-433475947 tempest-SecurityGroupsTestJSON-433475947-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:20:53.681573 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-5fd4340e-1c2a-4091-8cc5-54daee2c4c82 tempest-SecurityGroupsTestJSON-433475947 tempest-SecurityGroupsTestJSON-433475947-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:20:53.756987 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-5fd4340e-1c2a-4091-8cc5-54daee2c4c82 tempest-SecurityGroupsTestJSON-433475947 tempest-SecurityGroupsTestJSON-433475947-project] [instance: 8cded6c0-70d3-4172-8c68-d7632a3f503f] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='87b0fec9-a948-41f9-a3c0-f0a6a5a8f1e7',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120658) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 18 22:20:53.758954 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-5fd4340e-1c2a-4091-8cc5-54daee2c4c82 tempest-SecurityGroupsTestJSON-433475947 tempest-SecurityGroupsTestJSON-433475947-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:20:53.759730 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-5fd4340e-1c2a-4091-8cc5-54daee2c4c82 tempest-SecurityGroupsTestJSON-433475947 tempest-SecurityGroupsTestJSON-433475947-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:20:53.761058 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-5fd4340e-1c2a-4091-8cc5-54daee2c4c82 tempest-SecurityGroupsTestJSON-433475947 tempest-SecurityGroupsTestJSON-433475947-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:20:54.015949 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-5fd4340e-1c2a-4091-8cc5-54daee2c4c82 tempest-SecurityGroupsTestJSON-433475947 tempest-SecurityGroupsTestJSON-433475947-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:20:54.016355 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-5fd4340e-1c2a-4091-8cc5-54daee2c4c82 tempest-SecurityGroupsTestJSON-433475947 tempest-SecurityGroupsTestJSON-433475947-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:20:54.016761 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-5fd4340e-1c2a-4091-8cc5-54daee2c4c82 tempest-SecurityGroupsTestJSON-433475947 tempest-SecurityGroupsTestJSON-433475947-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:20:54.017518 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-5fd4340e-1c2a-4091-8cc5-54daee2c4c82 tempest-SecurityGroupsTestJSON-433475947 tempest-SecurityGroupsTestJSON-433475947-project] Acquiring lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 18 22:20:54.017875 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-5fd4340e-1c2a-4091-8cc5-54daee2c4c82 tempest-SecurityGroupsTestJSON-433475947 tempest-SecurityGroupsTestJSON-433475947-project] Acquired lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 18 22:20:54.018609 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-07fb6793-cfe4-4a08-9ca7-1eee05eefb71 None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:20:54.019108 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-07fb6793-cfe4-4a08-9ca7-1eee05eefb71 None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:20:54.019999 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-07fb6793-cfe4-4a08-9ca7-1eee05eefb71 None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:20:54.021094 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-07fb6793-cfe4-4a08-9ca7-1eee05eefb71 None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:20:54.021878 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-07fb6793-cfe4-4a08-9ca7-1eee05eefb71 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:20:54.026453 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-07fb6793-cfe4-4a08-9ca7-1eee05eefb71 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:20:54.032271 n-d-853704-1 nova-conductor[120658]: ERROR nova.compute.rpcapi [None req-07fb6793-cfe4-4a08-9ca7-1eee05eefb71 None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 18 22:20:54.032666 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-07fb6793-cfe4-4a08-9ca7-1eee05eefb71 None None] Releasing lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 18 22:20:54.033624 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server [None req-07fb6793-cfe4-4a08-9ca7-1eee05eefb71 None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:20:54.033624 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:20:54.033624 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 18 22:20:54.033624 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 18 22:20:54.033624 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 18 22:20:54.033624 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 22:20:54.033624 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 18 22:20:54.033624 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 22:20:54.033624 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:20:54.033624 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 18 22:20:54.033624 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 18 22:20:54.033624 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 18 22:20:54.033624 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 18 22:20:54.033624 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 18 22:20:54.033624 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 18 22:20:54.033624 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 18 22:20:54.033624 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 18 22:20:54.034804 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 18 22:20:54.034804 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 18 22:20:54.034804 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 18 22:20:54.034804 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 18 22:20:54.034804 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 18 22:20:54.034804 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 18 22:20:54.034804 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:20:54.034804 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 22:21:03.924729 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=11 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:21:03.926635 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=13,nova_api:UPDATE=1,nova_api:DELETE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:21:04.029603 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:21:04.030357 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell1:SELECT=11,nova_cell1:INSERT=17 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:21:11.996445 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-e887b085-97b1-4dc1-9529-5d2066bc1760 tempest-ServerActionsTestJSON-1622474758 tempest-ServerActionsTestJSON-1622474758-project] Took 0.83 seconds to select destinations for 1 instance(s). {{(pid=120657) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 18 22:21:12.061143 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-e887b085-97b1-4dc1-9529-5d2066bc1760 tempest-ServerActionsTestJSON-1622474758 tempest-ServerActionsTestJSON-1622474758-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:21:12.061665 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-e887b085-97b1-4dc1-9529-5d2066bc1760 tempest-ServerActionsTestJSON-1622474758 tempest-ServerActionsTestJSON-1622474758-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.004s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:21:12.062081 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-e887b085-97b1-4dc1-9529-5d2066bc1760 tempest-ServerActionsTestJSON-1622474758 tempest-ServerActionsTestJSON-1622474758-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:21:12.359831 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-e887b085-97b1-4dc1-9529-5d2066bc1760 tempest-ServerActionsTestJSON-1622474758 tempest-ServerActionsTestJSON-1622474758-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:21:12.360666 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-e887b085-97b1-4dc1-9529-5d2066bc1760 tempest-ServerActionsTestJSON-1622474758 tempest-ServerActionsTestJSON-1622474758-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:21:12.361227 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-e887b085-97b1-4dc1-9529-5d2066bc1760 tempest-ServerActionsTestJSON-1622474758 tempest-ServerActionsTestJSON-1622474758-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:21:12.362083 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-e887b085-97b1-4dc1-9529-5d2066bc1760 tempest-ServerActionsTestJSON-1622474758 tempest-ServerActionsTestJSON-1622474758-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:21:12.362641 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-e887b085-97b1-4dc1-9529-5d2066bc1760 tempest-ServerActionsTestJSON-1622474758 tempest-ServerActionsTestJSON-1622474758-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:21:12.363182 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-e887b085-97b1-4dc1-9529-5d2066bc1760 tempest-ServerActionsTestJSON-1622474758 tempest-ServerActionsTestJSON-1622474758-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:21:12.394125 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-e887b085-97b1-4dc1-9529-5d2066bc1760 tempest-ServerActionsTestJSON-1622474758 tempest-ServerActionsTestJSON-1622474758-project] Getting quotas for project 0484481d3e8e4ccbba2cc98221722ff7. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 18 22:21:12.398841 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-e887b085-97b1-4dc1-9529-5d2066bc1760 tempest-ServerActionsTestJSON-1622474758 tempest-ServerActionsTestJSON-1622474758-project] Getting quotas for user f3b231842d20423b84273c933c01d3f9 and project 0484481d3e8e4ccbba2cc98221722ff7. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 18 22:21:12.410874 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-e887b085-97b1-4dc1-9529-5d2066bc1760 tempest-ServerActionsTestJSON-1622474758 tempest-ServerActionsTestJSON-1622474758-project] [instance: ae05e4b2-01d6-4e79-b5eb-9ea3bb3f5d1e] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120657) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 18 22:21:12.414706 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-e887b085-97b1-4dc1-9529-5d2066bc1760 tempest-ServerActionsTestJSON-1622474758 tempest-ServerActionsTestJSON-1622474758-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:21:12.415158 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-e887b085-97b1-4dc1-9529-5d2066bc1760 tempest-ServerActionsTestJSON-1622474758 tempest-ServerActionsTestJSON-1622474758-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:21:12.416269 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-e887b085-97b1-4dc1-9529-5d2066bc1760 tempest-ServerActionsTestJSON-1622474758 tempest-ServerActionsTestJSON-1622474758-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:21:12.459390 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-e887b085-97b1-4dc1-9529-5d2066bc1760 tempest-ServerActionsTestJSON-1622474758 tempest-ServerActionsTestJSON-1622474758-project] [instance: ae05e4b2-01d6-4e79-b5eb-9ea3bb3f5d1e] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='87b0fec9-a948-41f9-a3c0-f0a6a5a8f1e7',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120657) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 18 22:21:12.461046 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-e887b085-97b1-4dc1-9529-5d2066bc1760 tempest-ServerActionsTestJSON-1622474758 tempest-ServerActionsTestJSON-1622474758-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:21:12.461487 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-e887b085-97b1-4dc1-9529-5d2066bc1760 tempest-ServerActionsTestJSON-1622474758 tempest-ServerActionsTestJSON-1622474758-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:21:12.461888 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-e887b085-97b1-4dc1-9529-5d2066bc1760 tempest-ServerActionsTestJSON-1622474758 tempest-ServerActionsTestJSON-1622474758-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:21:12.722501 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-e887b085-97b1-4dc1-9529-5d2066bc1760 tempest-ServerActionsTestJSON-1622474758 tempest-ServerActionsTestJSON-1622474758-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:21:12.722863 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-e887b085-97b1-4dc1-9529-5d2066bc1760 tempest-ServerActionsTestJSON-1622474758 tempest-ServerActionsTestJSON-1622474758-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:21:12.723267 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-e887b085-97b1-4dc1-9529-5d2066bc1760 tempest-ServerActionsTestJSON-1622474758 tempest-ServerActionsTestJSON-1622474758-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:21:12.723935 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-e887b085-97b1-4dc1-9529-5d2066bc1760 tempest-ServerActionsTestJSON-1622474758 tempest-ServerActionsTestJSON-1622474758-project] Acquiring lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 18 22:21:12.724258 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-e887b085-97b1-4dc1-9529-5d2066bc1760 tempest-ServerActionsTestJSON-1622474758 tempest-ServerActionsTestJSON-1622474758-project] Acquired lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 18 22:21:12.725006 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-edbbba62-27a3-487a-b6eb-3331b3620f7c None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:21:12.726670 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-edbbba62-27a3-487a-b6eb-3331b3620f7c None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:21:12.726670 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-edbbba62-27a3-487a-b6eb-3331b3620f7c None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:21:12.726670 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-edbbba62-27a3-487a-b6eb-3331b3620f7c None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:21:12.726929 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-edbbba62-27a3-487a-b6eb-3331b3620f7c None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:21:12.727317 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-edbbba62-27a3-487a-b6eb-3331b3620f7c None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:21:12.736734 n-d-853704-1 nova-conductor[120657]: ERROR nova.compute.rpcapi [None req-edbbba62-27a3-487a-b6eb-3331b3620f7c None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 18 22:21:12.737132 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-edbbba62-27a3-487a-b6eb-3331b3620f7c None None] Releasing lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 18 22:21:12.738085 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server [None req-edbbba62-27a3-487a-b6eb-3331b3620f7c None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:21:12.738085 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:21:12.738085 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 18 22:21:12.738085 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 18 22:21:12.738085 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 18 22:21:12.738085 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 22:21:12.738085 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 18 22:21:12.738085 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 22:21:12.738085 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:21:12.738085 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 18 22:21:12.738085 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 18 22:21:12.738085 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 18 22:21:12.738085 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 18 22:21:12.738085 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 18 22:21:12.738085 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 18 22:21:12.738085 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 18 22:21:12.738085 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 18 22:21:12.739343 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 18 22:21:12.739343 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 18 22:21:12.739343 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 18 22:21:12.739343 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 18 22:21:12.739343 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 18 22:21:12.739343 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 18 22:21:12.739343 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:21:12.739343 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 22:21:13.758038 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-faae50e9-5e3f-47e0-a8e6-c48ab43da4d9 tempest-ServersNegativeTestJSON-2054695786 tempest-ServersNegativeTestJSON-2054695786-project] Took 0.71 seconds to select destinations for 1 instance(s). {{(pid=120658) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 18 22:21:13.792450 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-faae50e9-5e3f-47e0-a8e6-c48ab43da4d9 tempest-ServersNegativeTestJSON-2054695786 tempest-ServersNegativeTestJSON-2054695786-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:21:13.793453 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-faae50e9-5e3f-47e0-a8e6-c48ab43da4d9 tempest-ServersNegativeTestJSON-2054695786 tempest-ServersNegativeTestJSON-2054695786-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.003s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:21:13.794246 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-faae50e9-5e3f-47e0-a8e6-c48ab43da4d9 tempest-ServersNegativeTestJSON-2054695786 tempest-ServersNegativeTestJSON-2054695786-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:21:14.070811 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=15,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:21:14.073502 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-faae50e9-5e3f-47e0-a8e6-c48ab43da4d9 tempest-ServersNegativeTestJSON-2054695786 tempest-ServersNegativeTestJSON-2054695786-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:21:14.073946 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-faae50e9-5e3f-47e0-a8e6-c48ab43da4d9 tempest-ServersNegativeTestJSON-2054695786 tempest-ServersNegativeTestJSON-2054695786-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:21:14.075499 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-faae50e9-5e3f-47e0-a8e6-c48ab43da4d9 tempest-ServersNegativeTestJSON-2054695786 tempest-ServersNegativeTestJSON-2054695786-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:21:14.075499 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-faae50e9-5e3f-47e0-a8e6-c48ab43da4d9 tempest-ServersNegativeTestJSON-2054695786 tempest-ServersNegativeTestJSON-2054695786-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:21:14.075890 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-faae50e9-5e3f-47e0-a8e6-c48ab43da4d9 tempest-ServersNegativeTestJSON-2054695786 tempest-ServersNegativeTestJSON-2054695786-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:21:14.076037 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-faae50e9-5e3f-47e0-a8e6-c48ab43da4d9 tempest-ServersNegativeTestJSON-2054695786 tempest-ServersNegativeTestJSON-2054695786-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:21:14.092626 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-faae50e9-5e3f-47e0-a8e6-c48ab43da4d9 tempest-ServersNegativeTestJSON-2054695786 tempest-ServersNegativeTestJSON-2054695786-project] Getting quotas for project 38c79c30bfd9427d8b2eb4ec316915ff. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 18 22:21:14.098952 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-faae50e9-5e3f-47e0-a8e6-c48ab43da4d9 tempest-ServersNegativeTestJSON-2054695786 tempest-ServersNegativeTestJSON-2054695786-project] Getting quotas for user 2984b9d70a1248ed9c7bc7ec09655498 and project 38c79c30bfd9427d8b2eb4ec316915ff. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 18 22:21:14.115328 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-faae50e9-5e3f-47e0-a8e6-c48ab43da4d9 tempest-ServersNegativeTestJSON-2054695786 tempest-ServersNegativeTestJSON-2054695786-project] [instance: 782ab930-7cec-4f0c-be36-60f94d995923] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120658) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 18 22:21:14.116163 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-faae50e9-5e3f-47e0-a8e6-c48ab43da4d9 tempest-ServersNegativeTestJSON-2054695786 tempest-ServersNegativeTestJSON-2054695786-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:21:14.116870 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-faae50e9-5e3f-47e0-a8e6-c48ab43da4d9 tempest-ServersNegativeTestJSON-2054695786 tempest-ServersNegativeTestJSON-2054695786-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:21:14.117451 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-faae50e9-5e3f-47e0-a8e6-c48ab43da4d9 tempest-ServersNegativeTestJSON-2054695786 tempest-ServersNegativeTestJSON-2054695786-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:21:14.270633 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-faae50e9-5e3f-47e0-a8e6-c48ab43da4d9 tempest-ServersNegativeTestJSON-2054695786 tempest-ServersNegativeTestJSON-2054695786-project] [instance: 782ab930-7cec-4f0c-be36-60f94d995923] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='87b0fec9-a948-41f9-a3c0-f0a6a5a8f1e7',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120658) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 18 22:21:14.272465 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-faae50e9-5e3f-47e0-a8e6-c48ab43da4d9 tempest-ServersNegativeTestJSON-2054695786 tempest-ServersNegativeTestJSON-2054695786-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:21:14.273270 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-faae50e9-5e3f-47e0-a8e6-c48ab43da4d9 tempest-ServersNegativeTestJSON-2054695786 tempest-ServersNegativeTestJSON-2054695786-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:21:14.274154 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-faae50e9-5e3f-47e0-a8e6-c48ab43da4d9 tempest-ServersNegativeTestJSON-2054695786 tempest-ServersNegativeTestJSON-2054695786-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:21:14.568696 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-faae50e9-5e3f-47e0-a8e6-c48ab43da4d9 tempest-ServersNegativeTestJSON-2054695786 tempest-ServersNegativeTestJSON-2054695786-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:21:14.569340 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-faae50e9-5e3f-47e0-a8e6-c48ab43da4d9 tempest-ServersNegativeTestJSON-2054695786 tempest-ServersNegativeTestJSON-2054695786-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:21:14.569838 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-faae50e9-5e3f-47e0-a8e6-c48ab43da4d9 tempest-ServersNegativeTestJSON-2054695786 tempest-ServersNegativeTestJSON-2054695786-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:21:14.571508 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-faae50e9-5e3f-47e0-a8e6-c48ab43da4d9 tempest-ServersNegativeTestJSON-2054695786 tempest-ServersNegativeTestJSON-2054695786-project] Acquiring lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 18 22:21:14.572037 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-faae50e9-5e3f-47e0-a8e6-c48ab43da4d9 tempest-ServersNegativeTestJSON-2054695786 tempest-ServersNegativeTestJSON-2054695786-project] Acquired lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 18 22:21:14.573331 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-6e4b560a-f422-4921-91fc-cde549286495 None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:21:14.574344 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-6e4b560a-f422-4921-91fc-cde549286495 None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:21:14.574760 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-6e4b560a-f422-4921-91fc-cde549286495 None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:21:14.575636 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-6e4b560a-f422-4921-91fc-cde549286495 None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:21:14.576154 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-6e4b560a-f422-4921-91fc-cde549286495 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:21:14.577515 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-6e4b560a-f422-4921-91fc-cde549286495 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:21:14.593921 n-d-853704-1 nova-conductor[120658]: ERROR nova.compute.rpcapi [None req-6e4b560a-f422-4921-91fc-cde549286495 None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 18 22:21:14.594321 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-6e4b560a-f422-4921-91fc-cde549286495 None None] Releasing lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 18 22:21:14.599596 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server [None req-6e4b560a-f422-4921-91fc-cde549286495 None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:21:14.599596 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:21:14.599596 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 18 22:21:14.599596 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 18 22:21:14.599596 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 18 22:21:14.599596 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 22:21:14.599596 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 18 22:21:14.599596 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 22:21:14.599596 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:21:14.599596 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 18 22:21:14.599596 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 18 22:21:14.599596 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 18 22:21:14.599596 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 18 22:21:14.599596 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 18 22:21:14.599596 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 18 22:21:14.599596 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 18 22:21:14.599596 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 18 22:21:14.600992 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 18 22:21:14.600992 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 18 22:21:14.600992 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 18 22:21:14.600992 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 18 22:21:14.600992 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 18 22:21:14.600992 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 18 22:21:14.600992 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:21:14.600992 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 22:21:22.544645 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=13 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:21:22.604883 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=11,nova_api:UPDATE=1,nova_api:DELETE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:21:22.735324 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:21:22.739860 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell1:SELECT=11,nova_cell1:SAVEPOINT=1,nova_cell1:INSERT=18,nova_cell1:RELEASE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:21:23.029482 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=15,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:21:24.405349 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=13,nova_api:UPDATE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:21:24.480016 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=11,nova_api:DELETE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:21:24.582373 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:21:24.593562 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell1:SELECT=11,nova_cell1:SAVEPOINT=1,nova_cell1:INSERT=18,nova_cell1:RELEASE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:21:27.485180 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-9af204ce-2cae-4e75-8509-16c6d71401bd tempest-ServersTestManualDisk-1049292471 tempest-ServersTestManualDisk-1049292471-project] Took 0.68 seconds to select destinations for 1 instance(s). {{(pid=120657) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 18 22:21:27.506845 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-9af204ce-2cae-4e75-8509-16c6d71401bd tempest-ServersTestManualDisk-1049292471 tempest-ServersTestManualDisk-1049292471-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:21:27.507296 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-9af204ce-2cae-4e75-8509-16c6d71401bd tempest-ServersTestManualDisk-1049292471 tempest-ServersTestManualDisk-1049292471-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:21:27.507716 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-9af204ce-2cae-4e75-8509-16c6d71401bd tempest-ServersTestManualDisk-1049292471 tempest-ServersTestManualDisk-1049292471-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:21:27.692897 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-9af204ce-2cae-4e75-8509-16c6d71401bd tempest-ServersTestManualDisk-1049292471 tempest-ServersTestManualDisk-1049292471-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:21:27.693255 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-9af204ce-2cae-4e75-8509-16c6d71401bd tempest-ServersTestManualDisk-1049292471 tempest-ServersTestManualDisk-1049292471-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:21:27.693653 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-9af204ce-2cae-4e75-8509-16c6d71401bd tempest-ServersTestManualDisk-1049292471 tempest-ServersTestManualDisk-1049292471-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:21:27.694348 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-9af204ce-2cae-4e75-8509-16c6d71401bd tempest-ServersTestManualDisk-1049292471 tempest-ServersTestManualDisk-1049292471-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:21:27.694769 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-9af204ce-2cae-4e75-8509-16c6d71401bd tempest-ServersTestManualDisk-1049292471 tempest-ServersTestManualDisk-1049292471-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:21:27.695153 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-9af204ce-2cae-4e75-8509-16c6d71401bd tempest-ServersTestManualDisk-1049292471 tempest-ServersTestManualDisk-1049292471-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:21:27.710559 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-9af204ce-2cae-4e75-8509-16c6d71401bd tempest-ServersTestManualDisk-1049292471 tempest-ServersTestManualDisk-1049292471-project] Getting quotas for project 13996ae1291444808fb5b778d69ca1af. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 18 22:21:27.715528 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-9af204ce-2cae-4e75-8509-16c6d71401bd tempest-ServersTestManualDisk-1049292471 tempest-ServersTestManualDisk-1049292471-project] Getting quotas for user 52a5e6743c604d5982d76dd05bf31014 and project 13996ae1291444808fb5b778d69ca1af. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 18 22:21:27.726040 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-9af204ce-2cae-4e75-8509-16c6d71401bd tempest-ServersTestManualDisk-1049292471 tempest-ServersTestManualDisk-1049292471-project] [instance: 2e9c041c-8e4e-4fb2-b2e0-680fd993e630] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120657) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 18 22:21:27.726810 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-9af204ce-2cae-4e75-8509-16c6d71401bd tempest-ServersTestManualDisk-1049292471 tempest-ServersTestManualDisk-1049292471-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:21:27.727935 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-9af204ce-2cae-4e75-8509-16c6d71401bd tempest-ServersTestManualDisk-1049292471 tempest-ServersTestManualDisk-1049292471-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:21:27.728439 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-9af204ce-2cae-4e75-8509-16c6d71401bd tempest-ServersTestManualDisk-1049292471 tempest-ServersTestManualDisk-1049292471-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:21:27.784574 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-9af204ce-2cae-4e75-8509-16c6d71401bd tempest-ServersTestManualDisk-1049292471 tempest-ServersTestManualDisk-1049292471-project] [instance: 2e9c041c-8e4e-4fb2-b2e0-680fd993e630] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='87b0fec9-a948-41f9-a3c0-f0a6a5a8f1e7',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120657) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 18 22:21:27.786033 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-9af204ce-2cae-4e75-8509-16c6d71401bd tempest-ServersTestManualDisk-1049292471 tempest-ServersTestManualDisk-1049292471-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:21:27.786665 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-9af204ce-2cae-4e75-8509-16c6d71401bd tempest-ServersTestManualDisk-1049292471 tempest-ServersTestManualDisk-1049292471-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:21:27.787075 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-9af204ce-2cae-4e75-8509-16c6d71401bd tempest-ServersTestManualDisk-1049292471 tempest-ServersTestManualDisk-1049292471-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:21:27.980706 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-9af204ce-2cae-4e75-8509-16c6d71401bd tempest-ServersTestManualDisk-1049292471 tempest-ServersTestManualDisk-1049292471-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:21:27.981351 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-9af204ce-2cae-4e75-8509-16c6d71401bd tempest-ServersTestManualDisk-1049292471 tempest-ServersTestManualDisk-1049292471-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:21:27.981908 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-9af204ce-2cae-4e75-8509-16c6d71401bd tempest-ServersTestManualDisk-1049292471 tempest-ServersTestManualDisk-1049292471-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:21:27.982450 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-9af204ce-2cae-4e75-8509-16c6d71401bd tempest-ServersTestManualDisk-1049292471 tempest-ServersTestManualDisk-1049292471-project] Acquiring lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 18 22:21:27.983069 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-9af204ce-2cae-4e75-8509-16c6d71401bd tempest-ServersTestManualDisk-1049292471 tempest-ServersTestManualDisk-1049292471-project] Acquired lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 18 22:21:27.983960 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-29ab7039-165a-407b-88e0-30580643581f None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:21:27.991622 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-29ab7039-165a-407b-88e0-30580643581f None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:21:27.991622 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-29ab7039-165a-407b-88e0-30580643581f None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:21:27.991622 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-29ab7039-165a-407b-88e0-30580643581f None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:21:27.991622 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-29ab7039-165a-407b-88e0-30580643581f None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:21:27.991622 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-29ab7039-165a-407b-88e0-30580643581f None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:21:27.998038 n-d-853704-1 nova-conductor[120657]: ERROR nova.compute.rpcapi [None req-29ab7039-165a-407b-88e0-30580643581f None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 18 22:21:28.015828 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-29ab7039-165a-407b-88e0-30580643581f None None] Releasing lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 18 22:21:28.015828 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server [None req-29ab7039-165a-407b-88e0-30580643581f None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:21:28.015828 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:21:28.015828 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 18 22:21:28.015828 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 18 22:21:28.015828 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 18 22:21:28.015828 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 22:21:28.015828 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 18 22:21:28.015828 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 22:21:28.015828 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:21:28.015828 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 18 22:21:28.015828 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 18 22:21:28.015828 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 18 22:21:28.015828 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 18 22:21:28.015828 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 18 22:21:28.015828 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 18 22:21:28.017264 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 18 22:21:28.017264 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 18 22:21:28.017264 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 18 22:21:28.017264 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 18 22:21:28.017264 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 18 22:21:28.017264 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 18 22:21:28.017264 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 18 22:21:28.017264 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 18 22:21:28.017264 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:21:28.017264 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 22:21:37.923850 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=12 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:21:37.924672 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=12,nova_api:UPDATE=1,nova_api:DELETE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:21:37.997577 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:21:37.998379 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell1:SELECT=11,nova_cell1:SAVEPOINT=1,nova_cell1:INSERT=19,nova_cell1:RELEASE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:21:53.068766 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=12,nova_cell0:UPDATE=3 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:22:03.022149 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=14,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:22:07.070034 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-ee63cd99-5a9f-4429-b7b5-e630fc2fea5d tempest-ServersTestBootFromVolume-1101704075 tempest-ServersTestBootFromVolume-1101704075-project] Took 0.61 seconds to select destinations for 1 instance(s). {{(pid=120658) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 18 22:22:07.090392 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ee63cd99-5a9f-4429-b7b5-e630fc2fea5d tempest-ServersTestBootFromVolume-1101704075 tempest-ServersTestBootFromVolume-1101704075-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:22:07.090837 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ee63cd99-5a9f-4429-b7b5-e630fc2fea5d tempest-ServersTestBootFromVolume-1101704075 tempest-ServersTestBootFromVolume-1101704075-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:22:07.091233 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ee63cd99-5a9f-4429-b7b5-e630fc2fea5d tempest-ServersTestBootFromVolume-1101704075 tempest-ServersTestBootFromVolume-1101704075-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:22:07.314161 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ee63cd99-5a9f-4429-b7b5-e630fc2fea5d tempest-ServersTestBootFromVolume-1101704075 tempest-ServersTestBootFromVolume-1101704075-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:22:07.314594 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ee63cd99-5a9f-4429-b7b5-e630fc2fea5d tempest-ServersTestBootFromVolume-1101704075 tempest-ServersTestBootFromVolume-1101704075-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:22:07.314982 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ee63cd99-5a9f-4429-b7b5-e630fc2fea5d tempest-ServersTestBootFromVolume-1101704075 tempest-ServersTestBootFromVolume-1101704075-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:22:07.315633 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ee63cd99-5a9f-4429-b7b5-e630fc2fea5d tempest-ServersTestBootFromVolume-1101704075 tempest-ServersTestBootFromVolume-1101704075-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:22:07.316074 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ee63cd99-5a9f-4429-b7b5-e630fc2fea5d tempest-ServersTestBootFromVolume-1101704075 tempest-ServersTestBootFromVolume-1101704075-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:22:07.316461 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ee63cd99-5a9f-4429-b7b5-e630fc2fea5d tempest-ServersTestBootFromVolume-1101704075 tempest-ServersTestBootFromVolume-1101704075-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:22:07.332070 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-ee63cd99-5a9f-4429-b7b5-e630fc2fea5d tempest-ServersTestBootFromVolume-1101704075 tempest-ServersTestBootFromVolume-1101704075-project] Getting quotas for project e25f10b5590d42b3bcf4da8b6bff7af0. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 18 22:22:07.336058 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-ee63cd99-5a9f-4429-b7b5-e630fc2fea5d tempest-ServersTestBootFromVolume-1101704075 tempest-ServersTestBootFromVolume-1101704075-project] Getting quotas for user 9e404636ff2b41898d020719a73c9b33 and project e25f10b5590d42b3bcf4da8b6bff7af0. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 18 22:22:07.346623 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-ee63cd99-5a9f-4429-b7b5-e630fc2fea5d tempest-ServersTestBootFromVolume-1101704075 tempest-ServersTestBootFromVolume-1101704075-project] [instance: 8de8b46a-1f0e-4e61-80f4-8a1b77d42c6f] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120658) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 18 22:22:07.347356 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ee63cd99-5a9f-4429-b7b5-e630fc2fea5d tempest-ServersTestBootFromVolume-1101704075 tempest-ServersTestBootFromVolume-1101704075-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:22:07.347812 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ee63cd99-5a9f-4429-b7b5-e630fc2fea5d tempest-ServersTestBootFromVolume-1101704075 tempest-ServersTestBootFromVolume-1101704075-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:22:07.348206 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ee63cd99-5a9f-4429-b7b5-e630fc2fea5d tempest-ServersTestBootFromVolume-1101704075 tempest-ServersTestBootFromVolume-1101704075-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:22:07.417188 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-ee63cd99-5a9f-4429-b7b5-e630fc2fea5d tempest-ServersTestBootFromVolume-1101704075 tempest-ServersTestBootFromVolume-1101704075-project] [instance: 8de8b46a-1f0e-4e61-80f4-8a1b77d42c6f] block_device_mapping [BlockDeviceMapping(attachment_id=9eeda0d9-7a82-4d42-8334-1e44bebb4b89,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='volume',device_name=None,device_type=None,disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id=None,instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='volume',tag=None,updated_at=,uuid=,volume_id='cad0ef92-4393-4d16-a36c-85a470539c97',volume_size=1,volume_type=None)] {{(pid=120658) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 18 22:22:07.418426 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ee63cd99-5a9f-4429-b7b5-e630fc2fea5d tempest-ServersTestBootFromVolume-1101704075 tempest-ServersTestBootFromVolume-1101704075-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:22:07.418869 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ee63cd99-5a9f-4429-b7b5-e630fc2fea5d tempest-ServersTestBootFromVolume-1101704075 tempest-ServersTestBootFromVolume-1101704075-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:22:07.419273 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ee63cd99-5a9f-4429-b7b5-e630fc2fea5d tempest-ServersTestBootFromVolume-1101704075 tempest-ServersTestBootFromVolume-1101704075-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:22:07.675413 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ee63cd99-5a9f-4429-b7b5-e630fc2fea5d tempest-ServersTestBootFromVolume-1101704075 tempest-ServersTestBootFromVolume-1101704075-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:22:07.675888 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ee63cd99-5a9f-4429-b7b5-e630fc2fea5d tempest-ServersTestBootFromVolume-1101704075 tempest-ServersTestBootFromVolume-1101704075-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:22:07.676283 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ee63cd99-5a9f-4429-b7b5-e630fc2fea5d tempest-ServersTestBootFromVolume-1101704075 tempest-ServersTestBootFromVolume-1101704075-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:22:07.676738 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ee63cd99-5a9f-4429-b7b5-e630fc2fea5d tempest-ServersTestBootFromVolume-1101704075 tempest-ServersTestBootFromVolume-1101704075-project] Acquiring lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 18 22:22:07.677100 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ee63cd99-5a9f-4429-b7b5-e630fc2fea5d tempest-ServersTestBootFromVolume-1101704075 tempest-ServersTestBootFromVolume-1101704075-project] Acquired lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 18 22:22:07.677957 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-99aa8230-7947-448e-a1b4-8f1eaf4a5595 None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:22:07.678370 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-99aa8230-7947-448e-a1b4-8f1eaf4a5595 None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:22:07.678757 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-99aa8230-7947-448e-a1b4-8f1eaf4a5595 None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:22:07.679401 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-99aa8230-7947-448e-a1b4-8f1eaf4a5595 None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:22:07.679849 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-99aa8230-7947-448e-a1b4-8f1eaf4a5595 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:22:07.684894 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-99aa8230-7947-448e-a1b4-8f1eaf4a5595 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:22:07.689617 n-d-853704-1 nova-conductor[120658]: ERROR nova.compute.rpcapi [None req-99aa8230-7947-448e-a1b4-8f1eaf4a5595 None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 18 22:22:07.690019 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-99aa8230-7947-448e-a1b4-8f1eaf4a5595 None None] Releasing lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 18 22:22:07.690956 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server [None req-99aa8230-7947-448e-a1b4-8f1eaf4a5595 None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:22:07.690956 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:22:07.690956 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 18 22:22:07.690956 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 18 22:22:07.690956 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 18 22:22:07.690956 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 22:22:07.690956 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 18 22:22:07.690956 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 22:22:07.690956 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:22:07.690956 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 18 22:22:07.690956 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 18 22:22:07.690956 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 18 22:22:07.690956 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 18 22:22:07.690956 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 18 22:22:07.690956 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 18 22:22:07.690956 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 18 22:22:07.690956 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 18 22:22:07.692275 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 18 22:22:07.692275 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 18 22:22:07.692275 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 18 22:22:07.692275 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 18 22:22:07.692275 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 18 22:22:07.692275 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 18 22:22:07.692275 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:22:07.692275 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 22:22:17.602867 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=14 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:22:17.603601 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=10,nova_api:UPDATE=1,nova_api:DELETE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:22:17.685645 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:22:17.687524 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell1:SELECT=11,nova_cell1:SAVEPOINT=1,nova_cell1:INSERT=18,nova_cell1:RELEASE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:22:23.030052 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=3,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:22:23.070307 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=11,nova_cell0:UPDATE=3 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:22:53.041843 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=6,nova_cell0:UPDATE=3 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:22:53.082447 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=6,nova_cell0:UPDATE=3 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:23:33.042587 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:23:33.082275 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:24:13.050390 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:24:13.075324 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:24:32.944712 n-d-853704-1 nova-conductor[120657]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 18 22:24:33.054626 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=3,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:24:33.077056 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=3,nova_cell0:UPDATE=2 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:24:56.619223 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-0feca7ab-597e-416a-9d0e-a51d03d66824 tempest-TenantUsagesTestJSON-1093647796 tempest-TenantUsagesTestJSON-1093647796-project] Took 0.64 seconds to select destinations for 1 instance(s). {{(pid=120657) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 18 22:24:56.641210 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-0feca7ab-597e-416a-9d0e-a51d03d66824 tempest-TenantUsagesTestJSON-1093647796 tempest-TenantUsagesTestJSON-1093647796-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:24:56.641648 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-0feca7ab-597e-416a-9d0e-a51d03d66824 tempest-TenantUsagesTestJSON-1093647796 tempest-TenantUsagesTestJSON-1093647796-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:24:56.642040 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-0feca7ab-597e-416a-9d0e-a51d03d66824 tempest-TenantUsagesTestJSON-1093647796 tempest-TenantUsagesTestJSON-1093647796-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:24:56.812261 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-0feca7ab-597e-416a-9d0e-a51d03d66824 tempest-TenantUsagesTestJSON-1093647796 tempest-TenantUsagesTestJSON-1093647796-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:24:56.815828 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-0feca7ab-597e-416a-9d0e-a51d03d66824 tempest-TenantUsagesTestJSON-1093647796 tempest-TenantUsagesTestJSON-1093647796-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:24:56.815828 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-0feca7ab-597e-416a-9d0e-a51d03d66824 tempest-TenantUsagesTestJSON-1093647796 tempest-TenantUsagesTestJSON-1093647796-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:24:56.815828 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-0feca7ab-597e-416a-9d0e-a51d03d66824 tempest-TenantUsagesTestJSON-1093647796 tempest-TenantUsagesTestJSON-1093647796-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:24:56.815828 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-0feca7ab-597e-416a-9d0e-a51d03d66824 tempest-TenantUsagesTestJSON-1093647796 tempest-TenantUsagesTestJSON-1093647796-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:24:56.816237 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-0feca7ab-597e-416a-9d0e-a51d03d66824 tempest-TenantUsagesTestJSON-1093647796 tempest-TenantUsagesTestJSON-1093647796-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:24:56.830376 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-0feca7ab-597e-416a-9d0e-a51d03d66824 tempest-TenantUsagesTestJSON-1093647796 tempest-TenantUsagesTestJSON-1093647796-project] Getting quotas for project d22bcf6070944d7da40e264ec3aa7857. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 18 22:24:56.834522 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-0feca7ab-597e-416a-9d0e-a51d03d66824 tempest-TenantUsagesTestJSON-1093647796 tempest-TenantUsagesTestJSON-1093647796-project] Getting quotas for user 016301ee3a444c9093ae54ad2ccac885 and project d22bcf6070944d7da40e264ec3aa7857. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 18 22:24:56.844827 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-0feca7ab-597e-416a-9d0e-a51d03d66824 tempest-TenantUsagesTestJSON-1093647796 tempest-TenantUsagesTestJSON-1093647796-project] [instance: 33150a65-5db6-48ff-824b-4cb9fe9dc854] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120657) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 18 22:24:56.845582 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-0feca7ab-597e-416a-9d0e-a51d03d66824 tempest-TenantUsagesTestJSON-1093647796 tempest-TenantUsagesTestJSON-1093647796-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:24:56.846012 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-0feca7ab-597e-416a-9d0e-a51d03d66824 tempest-TenantUsagesTestJSON-1093647796 tempest-TenantUsagesTestJSON-1093647796-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:24:56.846435 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-0feca7ab-597e-416a-9d0e-a51d03d66824 tempest-TenantUsagesTestJSON-1093647796 tempest-TenantUsagesTestJSON-1093647796-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:24:56.921312 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-0feca7ab-597e-416a-9d0e-a51d03d66824 tempest-TenantUsagesTestJSON-1093647796 tempest-TenantUsagesTestJSON-1093647796-project] [instance: 33150a65-5db6-48ff-824b-4cb9fe9dc854] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='87b0fec9-a948-41f9-a3c0-f0a6a5a8f1e7',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120657) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 18 22:24:56.922515 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-0feca7ab-597e-416a-9d0e-a51d03d66824 tempest-TenantUsagesTestJSON-1093647796 tempest-TenantUsagesTestJSON-1093647796-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:24:56.922942 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-0feca7ab-597e-416a-9d0e-a51d03d66824 tempest-TenantUsagesTestJSON-1093647796 tempest-TenantUsagesTestJSON-1093647796-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:24:56.923331 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-0feca7ab-597e-416a-9d0e-a51d03d66824 tempest-TenantUsagesTestJSON-1093647796 tempest-TenantUsagesTestJSON-1093647796-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:24:57.178569 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-0feca7ab-597e-416a-9d0e-a51d03d66824 tempest-TenantUsagesTestJSON-1093647796 tempest-TenantUsagesTestJSON-1093647796-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:24:57.179012 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-0feca7ab-597e-416a-9d0e-a51d03d66824 tempest-TenantUsagesTestJSON-1093647796 tempest-TenantUsagesTestJSON-1093647796-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:24:57.179409 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-0feca7ab-597e-416a-9d0e-a51d03d66824 tempest-TenantUsagesTestJSON-1093647796 tempest-TenantUsagesTestJSON-1093647796-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:24:57.179898 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-0feca7ab-597e-416a-9d0e-a51d03d66824 tempest-TenantUsagesTestJSON-1093647796 tempest-TenantUsagesTestJSON-1093647796-project] Acquiring lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 18 22:24:57.180246 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-0feca7ab-597e-416a-9d0e-a51d03d66824 tempest-TenantUsagesTestJSON-1093647796 tempest-TenantUsagesTestJSON-1093647796-project] Acquired lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 18 22:24:57.180974 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-de8456f1-8ecb-4715-8347-347659e81002 None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:24:57.181378 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-de8456f1-8ecb-4715-8347-347659e81002 None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:24:57.181774 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-de8456f1-8ecb-4715-8347-347659e81002 None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:24:57.182395 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-de8456f1-8ecb-4715-8347-347659e81002 None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:24:57.182797 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-de8456f1-8ecb-4715-8347-347659e81002 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:24:57.183170 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-de8456f1-8ecb-4715-8347-347659e81002 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:24:57.191571 n-d-853704-1 nova-conductor[120657]: ERROR nova.compute.rpcapi [None req-de8456f1-8ecb-4715-8347-347659e81002 None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 18 22:24:57.195917 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-de8456f1-8ecb-4715-8347-347659e81002 None None] Releasing lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 18 22:24:57.195917 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server [None req-de8456f1-8ecb-4715-8347-347659e81002 None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:24:57.195917 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:24:57.195917 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 18 22:24:57.195917 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 18 22:24:57.195917 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 18 22:24:57.195917 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 22:24:57.195917 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 18 22:24:57.195917 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 22:24:57.195917 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:24:57.195917 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 18 22:24:57.195917 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 18 22:24:57.195917 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 18 22:24:57.195917 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 18 22:24:57.195917 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 18 22:24:57.195917 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 18 22:24:57.197195 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 18 22:24:57.197195 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 18 22:24:57.197195 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 18 22:24:57.197195 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 18 22:24:57.197195 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 18 22:24:57.197195 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 18 22:24:57.197195 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 18 22:24:57.197195 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 18 22:24:57.197195 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:24:57.197195 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 22:25:01.240054 n-d-853704-1 nova-conductor[120658]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 18 22:25:07.095143 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=13,nova_api:UPDATE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:25:07.096009 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=11,nova_api:DELETE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:25:07.188826 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:25:07.189559 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell1:SELECT=11,nova_cell1:SAVEPOINT=1,nova_cell1:INSERT=18,nova_cell1:RELEASE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:25:13.056452 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=15,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:25:13.078051 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:25:33.084587 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=3,nova_cell0:UPDATE=2 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:25:43.058297 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=5,nova_cell0:UPDATE=3 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:25:50.114597 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-c9ad0b19-4d39-4c49-b3fb-b586390bad0b tempest-ServerPasswordTestJSON-1953745193 tempest-ServerPasswordTestJSON-1953745193-project] Took 0.60 seconds to select destinations for 1 instance(s). {{(pid=120658) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 18 22:25:50.139388 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-c9ad0b19-4d39-4c49-b3fb-b586390bad0b tempest-ServerPasswordTestJSON-1953745193 tempest-ServerPasswordTestJSON-1953745193-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:25:50.140141 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-c9ad0b19-4d39-4c49-b3fb-b586390bad0b tempest-ServerPasswordTestJSON-1953745193 tempest-ServerPasswordTestJSON-1953745193-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:25:50.140640 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-c9ad0b19-4d39-4c49-b3fb-b586390bad0b tempest-ServerPasswordTestJSON-1953745193 tempest-ServerPasswordTestJSON-1953745193-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:25:50.318921 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-c9ad0b19-4d39-4c49-b3fb-b586390bad0b tempest-ServerPasswordTestJSON-1953745193 tempest-ServerPasswordTestJSON-1953745193-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:25:50.319370 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-c9ad0b19-4d39-4c49-b3fb-b586390bad0b tempest-ServerPasswordTestJSON-1953745193 tempest-ServerPasswordTestJSON-1953745193-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:25:50.321598 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-c9ad0b19-4d39-4c49-b3fb-b586390bad0b tempest-ServerPasswordTestJSON-1953745193 tempest-ServerPasswordTestJSON-1953745193-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:25:50.322280 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-c9ad0b19-4d39-4c49-b3fb-b586390bad0b tempest-ServerPasswordTestJSON-1953745193 tempest-ServerPasswordTestJSON-1953745193-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:25:50.322694 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-c9ad0b19-4d39-4c49-b3fb-b586390bad0b tempest-ServerPasswordTestJSON-1953745193 tempest-ServerPasswordTestJSON-1953745193-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:25:50.323252 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-c9ad0b19-4d39-4c49-b3fb-b586390bad0b tempest-ServerPasswordTestJSON-1953745193 tempest-ServerPasswordTestJSON-1953745193-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:25:50.339793 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-c9ad0b19-4d39-4c49-b3fb-b586390bad0b tempest-ServerPasswordTestJSON-1953745193 tempest-ServerPasswordTestJSON-1953745193-project] Getting quotas for project e7c98d9b91904e27af9d5cfb44632f50. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 18 22:25:50.343874 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-c9ad0b19-4d39-4c49-b3fb-b586390bad0b tempest-ServerPasswordTestJSON-1953745193 tempest-ServerPasswordTestJSON-1953745193-project] Getting quotas for user d4e8a5cbf4824f55baa00deb67708531 and project e7c98d9b91904e27af9d5cfb44632f50. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 18 22:25:50.354996 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-c9ad0b19-4d39-4c49-b3fb-b586390bad0b tempest-ServerPasswordTestJSON-1953745193 tempest-ServerPasswordTestJSON-1953745193-project] [instance: 1cacd55f-6ee7-40ef-a1ed-607b4cc4a069] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120658) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 18 22:25:50.355807 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-c9ad0b19-4d39-4c49-b3fb-b586390bad0b tempest-ServerPasswordTestJSON-1953745193 tempest-ServerPasswordTestJSON-1953745193-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:25:50.356244 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-c9ad0b19-4d39-4c49-b3fb-b586390bad0b tempest-ServerPasswordTestJSON-1953745193 tempest-ServerPasswordTestJSON-1953745193-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:25:50.356637 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-c9ad0b19-4d39-4c49-b3fb-b586390bad0b tempest-ServerPasswordTestJSON-1953745193 tempest-ServerPasswordTestJSON-1953745193-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:25:50.422377 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-c9ad0b19-4d39-4c49-b3fb-b586390bad0b tempest-ServerPasswordTestJSON-1953745193 tempest-ServerPasswordTestJSON-1953745193-project] [instance: 1cacd55f-6ee7-40ef-a1ed-607b4cc4a069] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='87b0fec9-a948-41f9-a3c0-f0a6a5a8f1e7',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120658) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 18 22:25:50.423603 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-c9ad0b19-4d39-4c49-b3fb-b586390bad0b tempest-ServerPasswordTestJSON-1953745193 tempest-ServerPasswordTestJSON-1953745193-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:25:50.424089 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-c9ad0b19-4d39-4c49-b3fb-b586390bad0b tempest-ServerPasswordTestJSON-1953745193 tempest-ServerPasswordTestJSON-1953745193-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:25:50.424512 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-c9ad0b19-4d39-4c49-b3fb-b586390bad0b tempest-ServerPasswordTestJSON-1953745193 tempest-ServerPasswordTestJSON-1953745193-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:25:50.646233 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-c9ad0b19-4d39-4c49-b3fb-b586390bad0b tempest-ServerPasswordTestJSON-1953745193 tempest-ServerPasswordTestJSON-1953745193-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:25:50.646904 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-c9ad0b19-4d39-4c49-b3fb-b586390bad0b tempest-ServerPasswordTestJSON-1953745193 tempest-ServerPasswordTestJSON-1953745193-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:25:50.647296 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-c9ad0b19-4d39-4c49-b3fb-b586390bad0b tempest-ServerPasswordTestJSON-1953745193 tempest-ServerPasswordTestJSON-1953745193-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:25:50.647773 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-c9ad0b19-4d39-4c49-b3fb-b586390bad0b tempest-ServerPasswordTestJSON-1953745193 tempest-ServerPasswordTestJSON-1953745193-project] Acquiring lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 18 22:25:50.648126 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-c9ad0b19-4d39-4c49-b3fb-b586390bad0b tempest-ServerPasswordTestJSON-1953745193 tempest-ServerPasswordTestJSON-1953745193-project] Acquired lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 18 22:25:50.648884 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-68645b91-4078-4246-92c2-75712f86663e None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:25:50.649294 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-68645b91-4078-4246-92c2-75712f86663e None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:25:50.649672 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-68645b91-4078-4246-92c2-75712f86663e None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:25:50.650390 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-68645b91-4078-4246-92c2-75712f86663e None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:25:50.651711 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-68645b91-4078-4246-92c2-75712f86663e None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:25:50.652201 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-68645b91-4078-4246-92c2-75712f86663e None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:25:50.663543 n-d-853704-1 nova-conductor[120658]: ERROR nova.compute.rpcapi [None req-68645b91-4078-4246-92c2-75712f86663e None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 18 22:25:50.664211 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-68645b91-4078-4246-92c2-75712f86663e None None] Releasing lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 18 22:25:50.665248 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server [None req-68645b91-4078-4246-92c2-75712f86663e None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:25:50.665248 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:25:50.665248 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 18 22:25:50.665248 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 18 22:25:50.665248 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 18 22:25:50.665248 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 22:25:50.665248 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 18 22:25:50.665248 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 22:25:50.665248 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:25:50.665248 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 18 22:25:50.665248 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 18 22:25:50.665248 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 18 22:25:50.665248 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 18 22:25:50.665248 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 18 22:25:50.665248 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 18 22:25:50.665248 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 18 22:25:50.665248 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 18 22:25:50.666482 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 18 22:25:50.666482 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 18 22:25:50.666482 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 18 22:25:50.666482 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 18 22:25:50.666482 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 18 22:25:50.666482 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 18 22:25:50.666482 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:25:50.666482 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 22:26:00.555973 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=12 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:26:00.556878 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=12,nova_api:UPDATE=1,nova_api:DELETE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:26:00.660687 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:26:00.663398 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell1:SELECT=11,nova_cell1:SAVEPOINT=1,nova_cell1:INSERT=18,nova_cell1:RELEASE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:26:03.099768 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=12,nova_cell0:UPDATE=3 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:26:13.072490 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=6,nova_cell0:UPDATE=3 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:26:20.073123 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-c0da86b8-e783-44b8-a152-fd3b1c77817d tempest-MigrationsAdminTest-1637837745 tempest-MigrationsAdminTest-1637837745-project] Took 0.37 seconds to select destinations for 1 instance(s). {{(pid=120657) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 18 22:26:20.093191 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c0da86b8-e783-44b8-a152-fd3b1c77817d tempest-MigrationsAdminTest-1637837745 tempest-MigrationsAdminTest-1637837745-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:26:20.093650 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c0da86b8-e783-44b8-a152-fd3b1c77817d tempest-MigrationsAdminTest-1637837745 tempest-MigrationsAdminTest-1637837745-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:26:20.094033 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c0da86b8-e783-44b8-a152-fd3b1c77817d tempest-MigrationsAdminTest-1637837745 tempest-MigrationsAdminTest-1637837745-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:26:20.209213 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c0da86b8-e783-44b8-a152-fd3b1c77817d tempest-MigrationsAdminTest-1637837745 tempest-MigrationsAdminTest-1637837745-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:26:20.209652 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c0da86b8-e783-44b8-a152-fd3b1c77817d tempest-MigrationsAdminTest-1637837745 tempest-MigrationsAdminTest-1637837745-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:26:20.210043 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c0da86b8-e783-44b8-a152-fd3b1c77817d tempest-MigrationsAdminTest-1637837745 tempest-MigrationsAdminTest-1637837745-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:26:20.211225 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c0da86b8-e783-44b8-a152-fd3b1c77817d tempest-MigrationsAdminTest-1637837745 tempest-MigrationsAdminTest-1637837745-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:26:20.211713 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c0da86b8-e783-44b8-a152-fd3b1c77817d tempest-MigrationsAdminTest-1637837745 tempest-MigrationsAdminTest-1637837745-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:26:20.212061 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c0da86b8-e783-44b8-a152-fd3b1c77817d tempest-MigrationsAdminTest-1637837745 tempest-MigrationsAdminTest-1637837745-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:26:20.229783 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-c0da86b8-e783-44b8-a152-fd3b1c77817d tempest-MigrationsAdminTest-1637837745 tempest-MigrationsAdminTest-1637837745-project] Getting quotas for project 55cd873dadd6496dbebea5c3290775eb. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 18 22:26:20.233759 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-c0da86b8-e783-44b8-a152-fd3b1c77817d tempest-MigrationsAdminTest-1637837745 tempest-MigrationsAdminTest-1637837745-project] Getting quotas for user caa4813c18704375b16dedceb822364f and project 55cd873dadd6496dbebea5c3290775eb. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 18 22:26:20.244250 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-c0da86b8-e783-44b8-a152-fd3b1c77817d tempest-MigrationsAdminTest-1637837745 tempest-MigrationsAdminTest-1637837745-project] [instance: ba52d300-fa7b-4114-a183-ec9f57b183d3] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120657) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 18 22:26:20.245024 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c0da86b8-e783-44b8-a152-fd3b1c77817d tempest-MigrationsAdminTest-1637837745 tempest-MigrationsAdminTest-1637837745-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:26:20.245458 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c0da86b8-e783-44b8-a152-fd3b1c77817d tempest-MigrationsAdminTest-1637837745 tempest-MigrationsAdminTest-1637837745-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:26:20.245847 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c0da86b8-e783-44b8-a152-fd3b1c77817d tempest-MigrationsAdminTest-1637837745 tempest-MigrationsAdminTest-1637837745-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:26:20.300790 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-c0da86b8-e783-44b8-a152-fd3b1c77817d tempest-MigrationsAdminTest-1637837745 tempest-MigrationsAdminTest-1637837745-project] [instance: ba52d300-fa7b-4114-a183-ec9f57b183d3] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='87b0fec9-a948-41f9-a3c0-f0a6a5a8f1e7',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120657) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 18 22:26:20.302192 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c0da86b8-e783-44b8-a152-fd3b1c77817d tempest-MigrationsAdminTest-1637837745 tempest-MigrationsAdminTest-1637837745-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:26:20.302646 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c0da86b8-e783-44b8-a152-fd3b1c77817d tempest-MigrationsAdminTest-1637837745 tempest-MigrationsAdminTest-1637837745-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:26:20.303045 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c0da86b8-e783-44b8-a152-fd3b1c77817d tempest-MigrationsAdminTest-1637837745 tempest-MigrationsAdminTest-1637837745-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:26:20.558019 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c0da86b8-e783-44b8-a152-fd3b1c77817d tempest-MigrationsAdminTest-1637837745 tempest-MigrationsAdminTest-1637837745-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:26:20.558380 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c0da86b8-e783-44b8-a152-fd3b1c77817d tempest-MigrationsAdminTest-1637837745 tempest-MigrationsAdminTest-1637837745-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:26:20.558767 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c0da86b8-e783-44b8-a152-fd3b1c77817d tempest-MigrationsAdminTest-1637837745 tempest-MigrationsAdminTest-1637837745-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:26:20.559216 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c0da86b8-e783-44b8-a152-fd3b1c77817d tempest-MigrationsAdminTest-1637837745 tempest-MigrationsAdminTest-1637837745-project] Acquiring lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 18 22:26:20.559561 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c0da86b8-e783-44b8-a152-fd3b1c77817d tempest-MigrationsAdminTest-1637837745 tempest-MigrationsAdminTest-1637837745-project] Acquired lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 18 22:26:20.560349 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-bd46ff64-61cb-4996-84bb-74e3b7b17711 None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:26:20.561718 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-bd46ff64-61cb-4996-84bb-74e3b7b17711 None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:26:20.562099 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-bd46ff64-61cb-4996-84bb-74e3b7b17711 None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:26:20.562738 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-bd46ff64-61cb-4996-84bb-74e3b7b17711 None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:26:20.563153 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-bd46ff64-61cb-4996-84bb-74e3b7b17711 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:26:20.563818 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-bd46ff64-61cb-4996-84bb-74e3b7b17711 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:26:20.571402 n-d-853704-1 nova-conductor[120657]: ERROR nova.compute.rpcapi [None req-bd46ff64-61cb-4996-84bb-74e3b7b17711 None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 18 22:26:20.571808 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-bd46ff64-61cb-4996-84bb-74e3b7b17711 None None] Releasing lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 18 22:26:20.572704 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server [None req-bd46ff64-61cb-4996-84bb-74e3b7b17711 None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:26:20.572704 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:26:20.572704 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 18 22:26:20.572704 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 18 22:26:20.572704 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 18 22:26:20.572704 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 22:26:20.572704 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 18 22:26:20.572704 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 22:26:20.572704 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:26:20.572704 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 18 22:26:20.572704 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 18 22:26:20.572704 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 18 22:26:20.572704 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 18 22:26:20.572704 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 18 22:26:20.572704 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 18 22:26:20.572704 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 18 22:26:20.572704 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 18 22:26:20.574015 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 18 22:26:20.574015 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 18 22:26:20.574015 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 18 22:26:20.574015 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 18 22:26:20.574015 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 18 22:26:20.574015 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 18 22:26:20.574015 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:26:20.574015 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 22:26:30.479985 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=9,nova_api:UPDATE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:26:30.480733 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=15,nova_api:DELETE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:26:30.569481 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:26:30.570112 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell1:SELECT=11,nova_cell1:INSERT=17 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:26:33.083027 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=10,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:26:43.100709 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:26:53.094788 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=4,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:27:13.099702 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=4,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:27:13.109757 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=6,nova_cell0:UPDATE=3 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:27:33.125939 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=4,nova_cell0:UPDATE=2 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:27:36.756643 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-1bce88ec-50d5-42f9-98cd-83831ec73a4b tempest-ServerTagsTestJSON-2072411221 tempest-ServerTagsTestJSON-2072411221-project] Took 0.83 seconds to select destinations for 1 instance(s). {{(pid=120658) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 18 22:27:36.796544 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-1bce88ec-50d5-42f9-98cd-83831ec73a4b tempest-ServerTagsTestJSON-2072411221 tempest-ServerTagsTestJSON-2072411221-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:27:36.797862 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-1bce88ec-50d5-42f9-98cd-83831ec73a4b tempest-ServerTagsTestJSON-2072411221 tempest-ServerTagsTestJSON-2072411221-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:27:36.798473 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-1bce88ec-50d5-42f9-98cd-83831ec73a4b tempest-ServerTagsTestJSON-2072411221 tempest-ServerTagsTestJSON-2072411221-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:27:37.041312 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-1bce88ec-50d5-42f9-98cd-83831ec73a4b tempest-ServerTagsTestJSON-2072411221 tempest-ServerTagsTestJSON-2072411221-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:27:37.042185 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-1bce88ec-50d5-42f9-98cd-83831ec73a4b tempest-ServerTagsTestJSON-2072411221 tempest-ServerTagsTestJSON-2072411221-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:27:37.043421 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-1bce88ec-50d5-42f9-98cd-83831ec73a4b tempest-ServerTagsTestJSON-2072411221 tempest-ServerTagsTestJSON-2072411221-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:27:37.046001 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-1bce88ec-50d5-42f9-98cd-83831ec73a4b tempest-ServerTagsTestJSON-2072411221 tempest-ServerTagsTestJSON-2072411221-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:27:37.048965 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-1bce88ec-50d5-42f9-98cd-83831ec73a4b tempest-ServerTagsTestJSON-2072411221 tempest-ServerTagsTestJSON-2072411221-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:27:37.049622 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-1bce88ec-50d5-42f9-98cd-83831ec73a4b tempest-ServerTagsTestJSON-2072411221 tempest-ServerTagsTestJSON-2072411221-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.002s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:27:37.097474 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-1bce88ec-50d5-42f9-98cd-83831ec73a4b tempest-ServerTagsTestJSON-2072411221 tempest-ServerTagsTestJSON-2072411221-project] Getting quotas for project 1d8ed361f11c479bb76bdc35078464ab. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 18 22:27:37.102389 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-1bce88ec-50d5-42f9-98cd-83831ec73a4b tempest-ServerTagsTestJSON-2072411221 tempest-ServerTagsTestJSON-2072411221-project] Getting quotas for user 076c3dbfd5544f1dbcd69244ae1c1f16 and project 1d8ed361f11c479bb76bdc35078464ab. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 18 22:27:37.118314 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-1bce88ec-50d5-42f9-98cd-83831ec73a4b tempest-ServerTagsTestJSON-2072411221 tempest-ServerTagsTestJSON-2072411221-project] [instance: d925328d-7785-49f1-ab46-61366d5660b2] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120658) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 18 22:27:37.119458 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-1bce88ec-50d5-42f9-98cd-83831ec73a4b tempest-ServerTagsTestJSON-2072411221 tempest-ServerTagsTestJSON-2072411221-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:27:37.120082 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-1bce88ec-50d5-42f9-98cd-83831ec73a4b tempest-ServerTagsTestJSON-2072411221 tempest-ServerTagsTestJSON-2072411221-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:27:37.120614 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-1bce88ec-50d5-42f9-98cd-83831ec73a4b tempest-ServerTagsTestJSON-2072411221 tempest-ServerTagsTestJSON-2072411221-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:27:37.212719 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-1bce88ec-50d5-42f9-98cd-83831ec73a4b tempest-ServerTagsTestJSON-2072411221 tempest-ServerTagsTestJSON-2072411221-project] [instance: d925328d-7785-49f1-ab46-61366d5660b2] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='87b0fec9-a948-41f9-a3c0-f0a6a5a8f1e7',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120658) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 18 22:27:37.214381 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-1bce88ec-50d5-42f9-98cd-83831ec73a4b tempest-ServerTagsTestJSON-2072411221 tempest-ServerTagsTestJSON-2072411221-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:27:37.215167 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-1bce88ec-50d5-42f9-98cd-83831ec73a4b tempest-ServerTagsTestJSON-2072411221 tempest-ServerTagsTestJSON-2072411221-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:27:37.216275 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-1bce88ec-50d5-42f9-98cd-83831ec73a4b tempest-ServerTagsTestJSON-2072411221 tempest-ServerTagsTestJSON-2072411221-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:27:37.560484 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-1bce88ec-50d5-42f9-98cd-83831ec73a4b tempest-ServerTagsTestJSON-2072411221 tempest-ServerTagsTestJSON-2072411221-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:27:37.564421 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-1bce88ec-50d5-42f9-98cd-83831ec73a4b tempest-ServerTagsTestJSON-2072411221 tempest-ServerTagsTestJSON-2072411221-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.005s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:27:37.564421 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-1bce88ec-50d5-42f9-98cd-83831ec73a4b tempest-ServerTagsTestJSON-2072411221 tempest-ServerTagsTestJSON-2072411221-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:27:37.564421 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-1bce88ec-50d5-42f9-98cd-83831ec73a4b tempest-ServerTagsTestJSON-2072411221 tempest-ServerTagsTestJSON-2072411221-project] Acquiring lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 18 22:27:37.564421 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-1bce88ec-50d5-42f9-98cd-83831ec73a4b tempest-ServerTagsTestJSON-2072411221 tempest-ServerTagsTestJSON-2072411221-project] Acquired lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 18 22:27:37.565173 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ff4acc8e-7be3-4765-8807-08a66a282e7e None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:27:37.566928 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ff4acc8e-7be3-4765-8807-08a66a282e7e None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.002s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:27:37.567965 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ff4acc8e-7be3-4765-8807-08a66a282e7e None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:27:37.568983 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ff4acc8e-7be3-4765-8807-08a66a282e7e None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:27:37.569668 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ff4acc8e-7be3-4765-8807-08a66a282e7e None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:27:37.570342 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ff4acc8e-7be3-4765-8807-08a66a282e7e None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:27:37.582479 n-d-853704-1 nova-conductor[120658]: ERROR nova.compute.rpcapi [None req-ff4acc8e-7be3-4765-8807-08a66a282e7e None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 18 22:27:37.583129 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ff4acc8e-7be3-4765-8807-08a66a282e7e None None] Releasing lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 18 22:27:37.584818 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server [None req-ff4acc8e-7be3-4765-8807-08a66a282e7e None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:27:37.584818 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:27:37.584818 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 18 22:27:37.584818 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 18 22:27:37.584818 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 18 22:27:37.584818 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 22:27:37.584818 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 18 22:27:37.584818 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 22:27:37.584818 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:27:37.584818 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 18 22:27:37.584818 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 18 22:27:37.584818 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 18 22:27:37.584818 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 18 22:27:37.584818 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 18 22:27:37.584818 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 18 22:27:37.584818 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 18 22:27:37.584818 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 18 22:27:37.586214 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 18 22:27:37.586214 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 18 22:27:37.586214 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 18 22:27:37.586214 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 18 22:27:37.586214 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 18 22:27:37.586214 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 18 22:27:37.586214 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:27:37.586214 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 22:27:43.371014 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-6dbfae27-ad5d-4241-b31c-f7ef6bb7ad24 tempest-AttachVolumeShelveTestJSON-783903826 tempest-AttachVolumeShelveTestJSON-783903826-project] Took 0.83 seconds to select destinations for 1 instance(s). {{(pid=120657) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 18 22:27:43.404491 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-6dbfae27-ad5d-4241-b31c-f7ef6bb7ad24 tempest-AttachVolumeShelveTestJSON-783903826 tempest-AttachVolumeShelveTestJSON-783903826-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:27:43.405154 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-6dbfae27-ad5d-4241-b31c-f7ef6bb7ad24 tempest-AttachVolumeShelveTestJSON-783903826 tempest-AttachVolumeShelveTestJSON-783903826-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:27:43.406952 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-6dbfae27-ad5d-4241-b31c-f7ef6bb7ad24 tempest-AttachVolumeShelveTestJSON-783903826 tempest-AttachVolumeShelveTestJSON-783903826-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:27:43.615144 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:27:43.618312 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-6dbfae27-ad5d-4241-b31c-f7ef6bb7ad24 tempest-AttachVolumeShelveTestJSON-783903826 tempest-AttachVolumeShelveTestJSON-783903826-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:27:43.619009 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-6dbfae27-ad5d-4241-b31c-f7ef6bb7ad24 tempest-AttachVolumeShelveTestJSON-783903826 tempest-AttachVolumeShelveTestJSON-783903826-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:27:43.619607 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-6dbfae27-ad5d-4241-b31c-f7ef6bb7ad24 tempest-AttachVolumeShelveTestJSON-783903826 tempest-AttachVolumeShelveTestJSON-783903826-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:27:43.620877 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-6dbfae27-ad5d-4241-b31c-f7ef6bb7ad24 tempest-AttachVolumeShelveTestJSON-783903826 tempest-AttachVolumeShelveTestJSON-783903826-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:27:43.621654 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-6dbfae27-ad5d-4241-b31c-f7ef6bb7ad24 tempest-AttachVolumeShelveTestJSON-783903826 tempest-AttachVolumeShelveTestJSON-783903826-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:27:43.622095 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-6dbfae27-ad5d-4241-b31c-f7ef6bb7ad24 tempest-AttachVolumeShelveTestJSON-783903826 tempest-AttachVolumeShelveTestJSON-783903826-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:27:43.680385 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-6dbfae27-ad5d-4241-b31c-f7ef6bb7ad24 tempest-AttachVolumeShelveTestJSON-783903826 tempest-AttachVolumeShelveTestJSON-783903826-project] Getting quotas for project 9490452f4e584b88a243d315381fa891. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 18 22:27:43.694384 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-6dbfae27-ad5d-4241-b31c-f7ef6bb7ad24 tempest-AttachVolumeShelveTestJSON-783903826 tempest-AttachVolumeShelveTestJSON-783903826-project] Getting quotas for user b9b86470f04244d3b60ab4b3d42f3539 and project 9490452f4e584b88a243d315381fa891. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 18 22:27:43.719212 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-6dbfae27-ad5d-4241-b31c-f7ef6bb7ad24 tempest-AttachVolumeShelveTestJSON-783903826 tempest-AttachVolumeShelveTestJSON-783903826-project] [instance: 0e3b38a4-2bd4-479d-ae49-8e3ae1afe0a0] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120657) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 18 22:27:43.732567 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-6dbfae27-ad5d-4241-b31c-f7ef6bb7ad24 tempest-AttachVolumeShelveTestJSON-783903826 tempest-AttachVolumeShelveTestJSON-783903826-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:27:43.733020 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-6dbfae27-ad5d-4241-b31c-f7ef6bb7ad24 tempest-AttachVolumeShelveTestJSON-783903826 tempest-AttachVolumeShelveTestJSON-783903826-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:27:43.733412 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-6dbfae27-ad5d-4241-b31c-f7ef6bb7ad24 tempest-AttachVolumeShelveTestJSON-783903826 tempest-AttachVolumeShelveTestJSON-783903826-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:27:43.836856 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-6dbfae27-ad5d-4241-b31c-f7ef6bb7ad24 tempest-AttachVolumeShelveTestJSON-783903826 tempest-AttachVolumeShelveTestJSON-783903826-project] [instance: 0e3b38a4-2bd4-479d-ae49-8e3ae1afe0a0] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='87b0fec9-a948-41f9-a3c0-f0a6a5a8f1e7',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120657) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 18 22:27:43.838416 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-6dbfae27-ad5d-4241-b31c-f7ef6bb7ad24 tempest-AttachVolumeShelveTestJSON-783903826 tempest-AttachVolumeShelveTestJSON-783903826-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:27:43.839244 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-6dbfae27-ad5d-4241-b31c-f7ef6bb7ad24 tempest-AttachVolumeShelveTestJSON-783903826 tempest-AttachVolumeShelveTestJSON-783903826-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:27:43.840555 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-6dbfae27-ad5d-4241-b31c-f7ef6bb7ad24 tempest-AttachVolumeShelveTestJSON-783903826 tempest-AttachVolumeShelveTestJSON-783903826-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:27:44.134333 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-6dbfae27-ad5d-4241-b31c-f7ef6bb7ad24 tempest-AttachVolumeShelveTestJSON-783903826 tempest-AttachVolumeShelveTestJSON-783903826-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:27:44.135058 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-6dbfae27-ad5d-4241-b31c-f7ef6bb7ad24 tempest-AttachVolumeShelveTestJSON-783903826 tempest-AttachVolumeShelveTestJSON-783903826-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:27:44.135797 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-6dbfae27-ad5d-4241-b31c-f7ef6bb7ad24 tempest-AttachVolumeShelveTestJSON-783903826 tempest-AttachVolumeShelveTestJSON-783903826-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:27:44.136426 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-6dbfae27-ad5d-4241-b31c-f7ef6bb7ad24 tempest-AttachVolumeShelveTestJSON-783903826 tempest-AttachVolumeShelveTestJSON-783903826-project] Acquiring lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 18 22:27:44.137630 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-6dbfae27-ad5d-4241-b31c-f7ef6bb7ad24 tempest-AttachVolumeShelveTestJSON-783903826 tempest-AttachVolumeShelveTestJSON-783903826-project] Acquired lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 18 22:27:44.138833 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-8b56c505-75c7-41fe-b358-5642d1fe363b None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:27:44.139439 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-8b56c505-75c7-41fe-b358-5642d1fe363b None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:27:44.140104 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-8b56c505-75c7-41fe-b358-5642d1fe363b None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:27:44.146415 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-8b56c505-75c7-41fe-b358-5642d1fe363b None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:27:44.146415 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-8b56c505-75c7-41fe-b358-5642d1fe363b None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:27:44.146415 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-8b56c505-75c7-41fe-b358-5642d1fe363b None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:27:44.160012 n-d-853704-1 nova-conductor[120657]: ERROR nova.compute.rpcapi [None req-8b56c505-75c7-41fe-b358-5642d1fe363b None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 18 22:27:44.160400 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-8b56c505-75c7-41fe-b358-5642d1fe363b None None] Releasing lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 18 22:27:44.161307 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server [None req-8b56c505-75c7-41fe-b358-5642d1fe363b None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:27:44.161307 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:27:44.161307 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 18 22:27:44.161307 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 18 22:27:44.161307 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 18 22:27:44.161307 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 22:27:44.161307 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 18 22:27:44.161307 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 22:27:44.161307 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:27:44.161307 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 18 22:27:44.161307 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 18 22:27:44.161307 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 18 22:27:44.161307 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 18 22:27:44.161307 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 18 22:27:44.161307 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 18 22:27:44.161307 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 18 22:27:44.161307 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 18 22:27:44.162368 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 18 22:27:44.162368 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 18 22:27:44.162368 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 18 22:27:44.162368 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 18 22:27:44.162368 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 18 22:27:44.162368 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 18 22:27:44.162368 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:27:44.162368 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 22:27:47.417089 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=13,nova_api:UPDATE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:27:47.418218 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=11,nova_api:DELETE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:27:47.582990 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:27:47.583793 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell1:SELECT=11,nova_cell1:SAVEPOINT=1,nova_cell1:INSERT=18,nova_cell1:RELEASE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:27:54.040095 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=12,nova_api:UPDATE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:27:54.040857 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=12,nova_api:DELETE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:27:54.160020 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:27:54.163579 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell1:SELECT=11,nova_cell1:SAVEPOINT=1,nova_cell1:INSERT=18,nova_cell1:RELEASE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:28:13.153920 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=15,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:28:21.405091 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-b11e5d99-606a-4209-9fd5-fb8a1901ac65 tempest-ListServerFiltersTestJSON-664663457 tempest-ListServerFiltersTestJSON-664663457-project] Took 0.75 seconds to select destinations for 1 instance(s). {{(pid=120658) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 18 22:28:21.497644 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-b11e5d99-606a-4209-9fd5-fb8a1901ac65 tempest-ListServerFiltersTestJSON-664663457 tempest-ListServerFiltersTestJSON-664663457-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:28:21.502142 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-b11e5d99-606a-4209-9fd5-fb8a1901ac65 tempest-ListServerFiltersTestJSON-664663457 tempest-ListServerFiltersTestJSON-664663457-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.004s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:28:21.504409 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-b11e5d99-606a-4209-9fd5-fb8a1901ac65 tempest-ListServerFiltersTestJSON-664663457 tempest-ListServerFiltersTestJSON-664663457-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.002s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:28:21.784063 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-b11e5d99-606a-4209-9fd5-fb8a1901ac65 tempest-ListServerFiltersTestJSON-664663457 tempest-ListServerFiltersTestJSON-664663457-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:28:21.784839 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-b11e5d99-606a-4209-9fd5-fb8a1901ac65 tempest-ListServerFiltersTestJSON-664663457 tempest-ListServerFiltersTestJSON-664663457-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:28:21.785688 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-b11e5d99-606a-4209-9fd5-fb8a1901ac65 tempest-ListServerFiltersTestJSON-664663457 tempest-ListServerFiltersTestJSON-664663457-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:28:21.787232 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-b11e5d99-606a-4209-9fd5-fb8a1901ac65 tempest-ListServerFiltersTestJSON-664663457 tempest-ListServerFiltersTestJSON-664663457-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:28:21.788051 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-b11e5d99-606a-4209-9fd5-fb8a1901ac65 tempest-ListServerFiltersTestJSON-664663457 tempest-ListServerFiltersTestJSON-664663457-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:28:21.788959 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-b11e5d99-606a-4209-9fd5-fb8a1901ac65 tempest-ListServerFiltersTestJSON-664663457 tempest-ListServerFiltersTestJSON-664663457-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:28:21.808153 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-b11e5d99-606a-4209-9fd5-fb8a1901ac65 tempest-ListServerFiltersTestJSON-664663457 tempest-ListServerFiltersTestJSON-664663457-project] Getting quotas for project 33c9e7c7845d41bbac57822d053e8090. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 18 22:28:21.814885 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-b11e5d99-606a-4209-9fd5-fb8a1901ac65 tempest-ListServerFiltersTestJSON-664663457 tempest-ListServerFiltersTestJSON-664663457-project] Getting quotas for user 6e15352b9c6c4c91a787168a99dc4c87 and project 33c9e7c7845d41bbac57822d053e8090. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 18 22:28:21.848512 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-b11e5d99-606a-4209-9fd5-fb8a1901ac65 tempest-ListServerFiltersTestJSON-664663457 tempest-ListServerFiltersTestJSON-664663457-project] [instance: a7def17c-60a6-4df8-8d38-aea4b6933b40] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120658) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 18 22:28:21.849642 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-b11e5d99-606a-4209-9fd5-fb8a1901ac65 tempest-ListServerFiltersTestJSON-664663457 tempest-ListServerFiltersTestJSON-664663457-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:28:21.850280 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-b11e5d99-606a-4209-9fd5-fb8a1901ac65 tempest-ListServerFiltersTestJSON-664663457 tempest-ListServerFiltersTestJSON-664663457-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:28:21.851081 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-b11e5d99-606a-4209-9fd5-fb8a1901ac65 tempest-ListServerFiltersTestJSON-664663457 tempest-ListServerFiltersTestJSON-664663457-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:28:21.938392 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-b11e5d99-606a-4209-9fd5-fb8a1901ac65 tempest-ListServerFiltersTestJSON-664663457 tempest-ListServerFiltersTestJSON-664663457-project] [instance: a7def17c-60a6-4df8-8d38-aea4b6933b40] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='87b0fec9-a948-41f9-a3c0-f0a6a5a8f1e7',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120658) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 18 22:28:21.940347 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-b11e5d99-606a-4209-9fd5-fb8a1901ac65 tempest-ListServerFiltersTestJSON-664663457 tempest-ListServerFiltersTestJSON-664663457-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:28:21.941011 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-b11e5d99-606a-4209-9fd5-fb8a1901ac65 tempest-ListServerFiltersTestJSON-664663457 tempest-ListServerFiltersTestJSON-664663457-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:28:21.941627 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-b11e5d99-606a-4209-9fd5-fb8a1901ac65 tempest-ListServerFiltersTestJSON-664663457 tempest-ListServerFiltersTestJSON-664663457-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:28:22.213368 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-b11e5d99-606a-4209-9fd5-fb8a1901ac65 tempest-ListServerFiltersTestJSON-664663457 tempest-ListServerFiltersTestJSON-664663457-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:28:22.213810 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-b11e5d99-606a-4209-9fd5-fb8a1901ac65 tempest-ListServerFiltersTestJSON-664663457 tempest-ListServerFiltersTestJSON-664663457-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:28:22.214255 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-b11e5d99-606a-4209-9fd5-fb8a1901ac65 tempest-ListServerFiltersTestJSON-664663457 tempest-ListServerFiltersTestJSON-664663457-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:28:22.214701 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-b11e5d99-606a-4209-9fd5-fb8a1901ac65 tempest-ListServerFiltersTestJSON-664663457 tempest-ListServerFiltersTestJSON-664663457-project] Acquiring lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 18 22:28:22.215315 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-b11e5d99-606a-4209-9fd5-fb8a1901ac65 tempest-ListServerFiltersTestJSON-664663457 tempest-ListServerFiltersTestJSON-664663457-project] Acquired lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 18 22:28:22.216303 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-7fa46009-e217-45e0-a0d0-d65ea8eea5df None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:28:22.226261 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-7fa46009-e217-45e0-a0d0-d65ea8eea5df None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:28:22.226261 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-7fa46009-e217-45e0-a0d0-d65ea8eea5df None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:28:22.226261 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-7fa46009-e217-45e0-a0d0-d65ea8eea5df None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:28:22.226261 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-7fa46009-e217-45e0-a0d0-d65ea8eea5df None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:28:22.226261 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-7fa46009-e217-45e0-a0d0-d65ea8eea5df None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:28:22.239462 n-d-853704-1 nova-conductor[120658]: ERROR nova.compute.rpcapi [None req-7fa46009-e217-45e0-a0d0-d65ea8eea5df None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 18 22:28:22.241234 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-7fa46009-e217-45e0-a0d0-d65ea8eea5df None None] Releasing lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 18 22:28:22.243490 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server [None req-7fa46009-e217-45e0-a0d0-d65ea8eea5df None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:28:22.243490 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:28:22.243490 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 18 22:28:22.243490 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 18 22:28:22.243490 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 18 22:28:22.243490 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 22:28:22.243490 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 18 22:28:22.243490 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 22:28:22.243490 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:28:22.243490 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 18 22:28:22.243490 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 18 22:28:22.243490 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 18 22:28:22.243490 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 18 22:28:22.243490 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 18 22:28:22.243490 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 18 22:28:22.243490 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 18 22:28:22.243490 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 18 22:28:22.244941 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 18 22:28:22.244941 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 18 22:28:22.244941 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 18 22:28:22.244941 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 18 22:28:22.244941 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 18 22:28:22.244941 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 18 22:28:22.244941 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:28:22.244941 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 22:28:23.093934 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=12,nova_cell0:UPDATE=3 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:28:23.190908 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-2d5b7bab-fa5c-49af-8b82-d8154a257b99 tempest-ListServerFiltersTestJSON-664663457 tempest-ListServerFiltersTestJSON-664663457-project] Took 0.65 seconds to select destinations for 1 instance(s). {{(pid=120657) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 18 22:28:23.228835 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-2d5b7bab-fa5c-49af-8b82-d8154a257b99 tempest-ListServerFiltersTestJSON-664663457 tempest-ListServerFiltersTestJSON-664663457-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:28:23.229287 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-2d5b7bab-fa5c-49af-8b82-d8154a257b99 tempest-ListServerFiltersTestJSON-664663457 tempest-ListServerFiltersTestJSON-664663457-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:28:23.229686 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-2d5b7bab-fa5c-49af-8b82-d8154a257b99 tempest-ListServerFiltersTestJSON-664663457 tempest-ListServerFiltersTestJSON-664663457-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:28:23.504065 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-2d5b7bab-fa5c-49af-8b82-d8154a257b99 tempest-ListServerFiltersTestJSON-664663457 tempest-ListServerFiltersTestJSON-664663457-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:28:23.504627 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-2d5b7bab-fa5c-49af-8b82-d8154a257b99 tempest-ListServerFiltersTestJSON-664663457 tempest-ListServerFiltersTestJSON-664663457-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:28:23.505741 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-2d5b7bab-fa5c-49af-8b82-d8154a257b99 tempest-ListServerFiltersTestJSON-664663457 tempest-ListServerFiltersTestJSON-664663457-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:28:23.507377 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-2d5b7bab-fa5c-49af-8b82-d8154a257b99 tempest-ListServerFiltersTestJSON-664663457 tempest-ListServerFiltersTestJSON-664663457-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:28:23.508077 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-2d5b7bab-fa5c-49af-8b82-d8154a257b99 tempest-ListServerFiltersTestJSON-664663457 tempest-ListServerFiltersTestJSON-664663457-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:28:23.508925 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-2d5b7bab-fa5c-49af-8b82-d8154a257b99 tempest-ListServerFiltersTestJSON-664663457 tempest-ListServerFiltersTestJSON-664663457-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:28:23.547354 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-2d5b7bab-fa5c-49af-8b82-d8154a257b99 tempest-ListServerFiltersTestJSON-664663457 tempest-ListServerFiltersTestJSON-664663457-project] Getting quotas for project 33c9e7c7845d41bbac57822d053e8090. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 18 22:28:23.563320 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-2d5b7bab-fa5c-49af-8b82-d8154a257b99 tempest-ListServerFiltersTestJSON-664663457 tempest-ListServerFiltersTestJSON-664663457-project] Getting quotas for user 6e15352b9c6c4c91a787168a99dc4c87 and project 33c9e7c7845d41bbac57822d053e8090. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 18 22:28:23.577069 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-2d5b7bab-fa5c-49af-8b82-d8154a257b99 tempest-ListServerFiltersTestJSON-664663457 tempest-ListServerFiltersTestJSON-664663457-project] [instance: f80f7dc8-28e5-4f8b-96bc-4d0c3e07670b] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120657) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 18 22:28:23.578112 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-2d5b7bab-fa5c-49af-8b82-d8154a257b99 tempest-ListServerFiltersTestJSON-664663457 tempest-ListServerFiltersTestJSON-664663457-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:28:23.578708 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-2d5b7bab-fa5c-49af-8b82-d8154a257b99 tempest-ListServerFiltersTestJSON-664663457 tempest-ListServerFiltersTestJSON-664663457-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:28:23.579397 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-2d5b7bab-fa5c-49af-8b82-d8154a257b99 tempest-ListServerFiltersTestJSON-664663457 tempest-ListServerFiltersTestJSON-664663457-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:28:23.655885 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-2d5b7bab-fa5c-49af-8b82-d8154a257b99 tempest-ListServerFiltersTestJSON-664663457 tempest-ListServerFiltersTestJSON-664663457-project] [instance: f80f7dc8-28e5-4f8b-96bc-4d0c3e07670b] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='87b0fec9-a948-41f9-a3c0-f0a6a5a8f1e7',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120657) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 18 22:28:23.666431 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-2d5b7bab-fa5c-49af-8b82-d8154a257b99 tempest-ListServerFiltersTestJSON-664663457 tempest-ListServerFiltersTestJSON-664663457-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:28:23.667041 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-2d5b7bab-fa5c-49af-8b82-d8154a257b99 tempest-ListServerFiltersTestJSON-664663457 tempest-ListServerFiltersTestJSON-664663457-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.005s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:28:23.667584 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-2d5b7bab-fa5c-49af-8b82-d8154a257b99 tempest-ListServerFiltersTestJSON-664663457 tempest-ListServerFiltersTestJSON-664663457-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:28:24.064462 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-2d5b7bab-fa5c-49af-8b82-d8154a257b99 tempest-ListServerFiltersTestJSON-664663457 tempest-ListServerFiltersTestJSON-664663457-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:28:24.064931 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-2d5b7bab-fa5c-49af-8b82-d8154a257b99 tempest-ListServerFiltersTestJSON-664663457 tempest-ListServerFiltersTestJSON-664663457-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.002s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:28:24.067277 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-2d5b7bab-fa5c-49af-8b82-d8154a257b99 tempest-ListServerFiltersTestJSON-664663457 tempest-ListServerFiltersTestJSON-664663457-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:28:24.067277 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-2d5b7bab-fa5c-49af-8b82-d8154a257b99 tempest-ListServerFiltersTestJSON-664663457 tempest-ListServerFiltersTestJSON-664663457-project] Acquiring lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 18 22:28:24.067277 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-2d5b7bab-fa5c-49af-8b82-d8154a257b99 tempest-ListServerFiltersTestJSON-664663457 tempest-ListServerFiltersTestJSON-664663457-project] Acquired lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 18 22:28:24.067277 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c2451360-276a-4232-9943-23d6fadb1ad6 None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:28:24.072865 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c2451360-276a-4232-9943-23d6fadb1ad6 None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:28:24.073283 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c2451360-276a-4232-9943-23d6fadb1ad6 None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.006s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:28:24.074018 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c2451360-276a-4232-9943-23d6fadb1ad6 None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:28:24.074441 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c2451360-276a-4232-9943-23d6fadb1ad6 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:28:24.074818 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c2451360-276a-4232-9943-23d6fadb1ad6 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:28:24.092645 n-d-853704-1 nova-conductor[120657]: ERROR nova.compute.rpcapi [None req-c2451360-276a-4232-9943-23d6fadb1ad6 None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 18 22:28:24.093452 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c2451360-276a-4232-9943-23d6fadb1ad6 None None] Releasing lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 18 22:28:24.094592 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server [None req-c2451360-276a-4232-9943-23d6fadb1ad6 None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:28:24.094592 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:28:24.094592 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 18 22:28:24.094592 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 18 22:28:24.094592 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 18 22:28:24.094592 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 22:28:24.094592 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 18 22:28:24.094592 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 22:28:24.094592 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:28:24.094592 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 18 22:28:24.094592 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 18 22:28:24.094592 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 18 22:28:24.094592 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 18 22:28:24.094592 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 18 22:28:24.094592 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 18 22:28:24.094592 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 18 22:28:24.094592 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 18 22:28:24.096392 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 18 22:28:24.096392 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 18 22:28:24.096392 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 18 22:28:24.096392 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 18 22:28:24.096392 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 18 22:28:24.096392 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 18 22:28:24.096392 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:28:24.096392 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 22:28:25.294309 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-5f31a90f-a852-48b6-8773-8d3034041730 tempest-ListServerFiltersTestJSON-664663457 tempest-ListServerFiltersTestJSON-664663457-project] Took 0.53 seconds to select destinations for 1 instance(s). {{(pid=120658) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 18 22:28:25.330973 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-5f31a90f-a852-48b6-8773-8d3034041730 tempest-ListServerFiltersTestJSON-664663457 tempest-ListServerFiltersTestJSON-664663457-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:28:25.331927 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-5f31a90f-a852-48b6-8773-8d3034041730 tempest-ListServerFiltersTestJSON-664663457 tempest-ListServerFiltersTestJSON-664663457-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:28:25.332421 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-5f31a90f-a852-48b6-8773-8d3034041730 tempest-ListServerFiltersTestJSON-664663457 tempest-ListServerFiltersTestJSON-664663457-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:28:25.516348 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-5f31a90f-a852-48b6-8773-8d3034041730 tempest-ListServerFiltersTestJSON-664663457 tempest-ListServerFiltersTestJSON-664663457-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:28:25.516928 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-5f31a90f-a852-48b6-8773-8d3034041730 tempest-ListServerFiltersTestJSON-664663457 tempest-ListServerFiltersTestJSON-664663457-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:28:25.517616 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-5f31a90f-a852-48b6-8773-8d3034041730 tempest-ListServerFiltersTestJSON-664663457 tempest-ListServerFiltersTestJSON-664663457-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:28:25.518376 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-5f31a90f-a852-48b6-8773-8d3034041730 tempest-ListServerFiltersTestJSON-664663457 tempest-ListServerFiltersTestJSON-664663457-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:28:25.518898 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-5f31a90f-a852-48b6-8773-8d3034041730 tempest-ListServerFiltersTestJSON-664663457 tempest-ListServerFiltersTestJSON-664663457-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:28:25.519416 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-5f31a90f-a852-48b6-8773-8d3034041730 tempest-ListServerFiltersTestJSON-664663457 tempest-ListServerFiltersTestJSON-664663457-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:28:25.538523 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-5f31a90f-a852-48b6-8773-8d3034041730 tempest-ListServerFiltersTestJSON-664663457 tempest-ListServerFiltersTestJSON-664663457-project] Getting quotas for project 33c9e7c7845d41bbac57822d053e8090. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 18 22:28:25.542772 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-5f31a90f-a852-48b6-8773-8d3034041730 tempest-ListServerFiltersTestJSON-664663457 tempest-ListServerFiltersTestJSON-664663457-project] Getting quotas for user 6e15352b9c6c4c91a787168a99dc4c87 and project 33c9e7c7845d41bbac57822d053e8090. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 18 22:28:25.555339 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-5f31a90f-a852-48b6-8773-8d3034041730 tempest-ListServerFiltersTestJSON-664663457 tempest-ListServerFiltersTestJSON-664663457-project] [instance: 5ecde0c1-1bd2-4eed-9dbb-122702422f26] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120658) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 18 22:28:25.556354 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-5f31a90f-a852-48b6-8773-8d3034041730 tempest-ListServerFiltersTestJSON-664663457 tempest-ListServerFiltersTestJSON-664663457-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:28:25.556786 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-5f31a90f-a852-48b6-8773-8d3034041730 tempest-ListServerFiltersTestJSON-664663457 tempest-ListServerFiltersTestJSON-664663457-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:28:25.557195 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-5f31a90f-a852-48b6-8773-8d3034041730 tempest-ListServerFiltersTestJSON-664663457 tempest-ListServerFiltersTestJSON-664663457-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:28:25.639992 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-5f31a90f-a852-48b6-8773-8d3034041730 tempest-ListServerFiltersTestJSON-664663457 tempest-ListServerFiltersTestJSON-664663457-project] [instance: 5ecde0c1-1bd2-4eed-9dbb-122702422f26] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='87b0fec9-a948-41f9-a3c0-f0a6a5a8f1e7',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120658) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 18 22:28:25.641215 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-5f31a90f-a852-48b6-8773-8d3034041730 tempest-ListServerFiltersTestJSON-664663457 tempest-ListServerFiltersTestJSON-664663457-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:28:25.641783 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-5f31a90f-a852-48b6-8773-8d3034041730 tempest-ListServerFiltersTestJSON-664663457 tempest-ListServerFiltersTestJSON-664663457-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:28:25.642176 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-5f31a90f-a852-48b6-8773-8d3034041730 tempest-ListServerFiltersTestJSON-664663457 tempest-ListServerFiltersTestJSON-664663457-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:28:26.048757 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-5f31a90f-a852-48b6-8773-8d3034041730 tempest-ListServerFiltersTestJSON-664663457 tempest-ListServerFiltersTestJSON-664663457-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:28:26.049199 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-5f31a90f-a852-48b6-8773-8d3034041730 tempest-ListServerFiltersTestJSON-664663457 tempest-ListServerFiltersTestJSON-664663457-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:28:26.050168 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-5f31a90f-a852-48b6-8773-8d3034041730 tempest-ListServerFiltersTestJSON-664663457 tempest-ListServerFiltersTestJSON-664663457-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:28:26.050634 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-5f31a90f-a852-48b6-8773-8d3034041730 tempest-ListServerFiltersTestJSON-664663457 tempest-ListServerFiltersTestJSON-664663457-project] Acquiring lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 18 22:28:26.050985 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-5f31a90f-a852-48b6-8773-8d3034041730 tempest-ListServerFiltersTestJSON-664663457 tempest-ListServerFiltersTestJSON-664663457-project] Acquired lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 18 22:28:26.051799 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-2f755133-c345-4571-9724-77c2ff9dd0fa None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:28:26.052178 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-2f755133-c345-4571-9724-77c2ff9dd0fa None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:28:26.057734 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-2f755133-c345-4571-9724-77c2ff9dd0fa None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:28:26.057734 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-2f755133-c345-4571-9724-77c2ff9dd0fa None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:28:26.057734 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-2f755133-c345-4571-9724-77c2ff9dd0fa None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:28:26.057734 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-2f755133-c345-4571-9724-77c2ff9dd0fa None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:28:26.065743 n-d-853704-1 nova-conductor[120658]: ERROR nova.compute.rpcapi [None req-2f755133-c345-4571-9724-77c2ff9dd0fa None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 18 22:28:26.066129 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-2f755133-c345-4571-9724-77c2ff9dd0fa None None] Releasing lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 18 22:28:26.067068 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server [None req-2f755133-c345-4571-9724-77c2ff9dd0fa None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:28:26.067068 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:28:26.067068 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 18 22:28:26.067068 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 18 22:28:26.067068 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 18 22:28:26.067068 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 22:28:26.067068 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 18 22:28:26.067068 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 22:28:26.067068 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:28:26.067068 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 18 22:28:26.067068 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 18 22:28:26.067068 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 18 22:28:26.067068 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 18 22:28:26.067068 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 18 22:28:26.067068 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 18 22:28:26.067068 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 18 22:28:26.067068 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 18 22:28:26.068422 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 18 22:28:26.068422 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 18 22:28:26.068422 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 18 22:28:26.068422 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 18 22:28:26.068422 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 18 22:28:26.068422 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 18 22:28:26.068422 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:28:26.068422 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 22:28:33.413001 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-e63a7a3f-cd92-4195-a2fa-a5ab1292ea46 tempest-TaggedBootDevicesTest-152730828 tempest-TaggedBootDevicesTest-152730828-project] Took 0.95 seconds to select destinations for 1 instance(s). {{(pid=120657) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 18 22:28:33.434543 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-e63a7a3f-cd92-4195-a2fa-a5ab1292ea46 tempest-TaggedBootDevicesTest-152730828 tempest-TaggedBootDevicesTest-152730828-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:28:33.434972 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-e63a7a3f-cd92-4195-a2fa-a5ab1292ea46 tempest-TaggedBootDevicesTest-152730828 tempest-TaggedBootDevicesTest-152730828-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:28:33.436070 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-e63a7a3f-cd92-4195-a2fa-a5ab1292ea46 tempest-TaggedBootDevicesTest-152730828 tempest-TaggedBootDevicesTest-152730828-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:28:33.759427 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-e63a7a3f-cd92-4195-a2fa-a5ab1292ea46 tempest-TaggedBootDevicesTest-152730828 tempest-TaggedBootDevicesTest-152730828-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:28:33.759912 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-e63a7a3f-cd92-4195-a2fa-a5ab1292ea46 tempest-TaggedBootDevicesTest-152730828 tempest-TaggedBootDevicesTest-152730828-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:28:33.760312 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-e63a7a3f-cd92-4195-a2fa-a5ab1292ea46 tempest-TaggedBootDevicesTest-152730828 tempest-TaggedBootDevicesTest-152730828-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:28:33.760978 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-e63a7a3f-cd92-4195-a2fa-a5ab1292ea46 tempest-TaggedBootDevicesTest-152730828 tempest-TaggedBootDevicesTest-152730828-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:28:33.761468 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-e63a7a3f-cd92-4195-a2fa-a5ab1292ea46 tempest-TaggedBootDevicesTest-152730828 tempest-TaggedBootDevicesTest-152730828-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:28:33.774557 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-e63a7a3f-cd92-4195-a2fa-a5ab1292ea46 tempest-TaggedBootDevicesTest-152730828 tempest-TaggedBootDevicesTest-152730828-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:28:33.791824 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-e63a7a3f-cd92-4195-a2fa-a5ab1292ea46 tempest-TaggedBootDevicesTest-152730828 tempest-TaggedBootDevicesTest-152730828-project] Getting quotas for project bd4d1daa6f8549f3aea014aa8d88482e. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 18 22:28:33.796119 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-e63a7a3f-cd92-4195-a2fa-a5ab1292ea46 tempest-TaggedBootDevicesTest-152730828 tempest-TaggedBootDevicesTest-152730828-project] Getting quotas for user e9ffef453ab147f9bfffde3c60f9fa4e and project bd4d1daa6f8549f3aea014aa8d88482e. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 18 22:28:33.807523 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-e63a7a3f-cd92-4195-a2fa-a5ab1292ea46 tempest-TaggedBootDevicesTest-152730828 tempest-TaggedBootDevicesTest-152730828-project] [instance: 18a7eccb-0a76-488c-a18d-8bcbe99a77e4] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120657) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 18 22:28:33.808268 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-e63a7a3f-cd92-4195-a2fa-a5ab1292ea46 tempest-TaggedBootDevicesTest-152730828 tempest-TaggedBootDevicesTest-152730828-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:28:33.808709 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-e63a7a3f-cd92-4195-a2fa-a5ab1292ea46 tempest-TaggedBootDevicesTest-152730828 tempest-TaggedBootDevicesTest-152730828-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:28:33.809106 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-e63a7a3f-cd92-4195-a2fa-a5ab1292ea46 tempest-TaggedBootDevicesTest-152730828 tempest-TaggedBootDevicesTest-152730828-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:28:33.892490 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-e63a7a3f-cd92-4195-a2fa-a5ab1292ea46 tempest-TaggedBootDevicesTest-152730828 tempest-TaggedBootDevicesTest-152730828-project] [instance: 18a7eccb-0a76-488c-a18d-8bcbe99a77e4] block_device_mapping [BlockDeviceMapping(attachment_id=99ec4c19-97cf-4616-93b0-de2aa50f3910,boot_index=0,connection_info=None,created_at=,delete_on_termination=False,deleted=,deleted_at=,destination_type='volume',device_name=None,device_type=None,disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id=None,instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='volume',tag='boot',updated_at=,uuid=,volume_id='2cfc63cf-dc3b-497e-bad1-35ad9aa1c768',volume_size=1,volume_type=None), BlockDeviceMapping(attachment_id=9e13443c-d7d2-4793-b355-8fd508271d15,boot_index=1,connection_info=None,created_at=,delete_on_termination=False,deleted=,deleted_at=,destination_type='volume',device_name=None,device_type=None,disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id=None,instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='volume',tag='other',updated_at=,uuid=,volume_id='965a465c-a7db-4d76-9e11-1d94416807e6',volume_size=1,volume_type=None), BlockDeviceMapping(attachment_id=3facbc62-a230-4325-9931-debf79e10081,boot_index=2,connection_info=None,created_at=,delete_on_termination=False,deleted=,deleted_at=,destination_type='volume',device_name=None,device_type=None,disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id=None,instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='volume',tag=None,updated_at=,uuid=,volume_id='511b3e33-786b-4730-880e-e84cf4d62487',volume_size=1,volume_type=None)] {{(pid=120657) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 18 22:28:33.895200 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-e63a7a3f-cd92-4195-a2fa-a5ab1292ea46 tempest-TaggedBootDevicesTest-152730828 tempest-TaggedBootDevicesTest-152730828-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:28:33.896025 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-e63a7a3f-cd92-4195-a2fa-a5ab1292ea46 tempest-TaggedBootDevicesTest-152730828 tempest-TaggedBootDevicesTest-152730828-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:28:33.896813 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-e63a7a3f-cd92-4195-a2fa-a5ab1292ea46 tempest-TaggedBootDevicesTest-152730828 tempest-TaggedBootDevicesTest-152730828-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:28:33.978277 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-e63a7a3f-cd92-4195-a2fa-a5ab1292ea46 tempest-TaggedBootDevicesTest-152730828 tempest-TaggedBootDevicesTest-152730828-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:28:33.979381 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-e63a7a3f-cd92-4195-a2fa-a5ab1292ea46 tempest-TaggedBootDevicesTest-152730828 tempest-TaggedBootDevicesTest-152730828-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:28:33.979884 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-e63a7a3f-cd92-4195-a2fa-a5ab1292ea46 tempest-TaggedBootDevicesTest-152730828 tempest-TaggedBootDevicesTest-152730828-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:28:34.067762 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-e63a7a3f-cd92-4195-a2fa-a5ab1292ea46 tempest-TaggedBootDevicesTest-152730828 tempest-TaggedBootDevicesTest-152730828-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:28:34.068401 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-e63a7a3f-cd92-4195-a2fa-a5ab1292ea46 tempest-TaggedBootDevicesTest-152730828 tempest-TaggedBootDevicesTest-152730828-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:28:34.069020 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-e63a7a3f-cd92-4195-a2fa-a5ab1292ea46 tempest-TaggedBootDevicesTest-152730828 tempest-TaggedBootDevicesTest-152730828-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:28:34.359750 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-e63a7a3f-cd92-4195-a2fa-a5ab1292ea46 tempest-TaggedBootDevicesTest-152730828 tempest-TaggedBootDevicesTest-152730828-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:28:34.360202 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-e63a7a3f-cd92-4195-a2fa-a5ab1292ea46 tempest-TaggedBootDevicesTest-152730828 tempest-TaggedBootDevicesTest-152730828-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:28:34.360601 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-e63a7a3f-cd92-4195-a2fa-a5ab1292ea46 tempest-TaggedBootDevicesTest-152730828 tempest-TaggedBootDevicesTest-152730828-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:28:34.361060 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-e63a7a3f-cd92-4195-a2fa-a5ab1292ea46 tempest-TaggedBootDevicesTest-152730828 tempest-TaggedBootDevicesTest-152730828-project] Acquiring lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 18 22:28:34.361417 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-e63a7a3f-cd92-4195-a2fa-a5ab1292ea46 tempest-TaggedBootDevicesTest-152730828 tempest-TaggedBootDevicesTest-152730828-project] Acquired lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 18 22:28:34.362145 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-803093c1-0816-4869-8c89-e2220b91de2e None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:28:34.362558 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-803093c1-0816-4869-8c89-e2220b91de2e None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:28:34.362948 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-803093c1-0816-4869-8c89-e2220b91de2e None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:28:34.363624 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-803093c1-0816-4869-8c89-e2220b91de2e None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:28:34.364071 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-803093c1-0816-4869-8c89-e2220b91de2e None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:28:34.364453 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-803093c1-0816-4869-8c89-e2220b91de2e None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:28:34.375962 n-d-853704-1 nova-conductor[120657]: ERROR nova.compute.rpcapi [None req-803093c1-0816-4869-8c89-e2220b91de2e None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 18 22:28:34.376462 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-803093c1-0816-4869-8c89-e2220b91de2e None None] Releasing lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 18 22:28:34.377868 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server [None req-803093c1-0816-4869-8c89-e2220b91de2e None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:28:34.377868 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:28:34.377868 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 18 22:28:34.377868 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 18 22:28:34.377868 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 18 22:28:34.377868 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 22:28:34.377868 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 18 22:28:34.377868 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 22:28:34.377868 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:28:34.377868 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 18 22:28:34.377868 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 18 22:28:34.377868 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 18 22:28:34.377868 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 18 22:28:34.377868 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 18 22:28:34.377868 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 18 22:28:34.377868 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 18 22:28:34.377868 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 18 22:28:34.377868 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 18 22:28:34.377868 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 18 22:28:34.377868 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 18 22:28:34.377868 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 18 22:28:34.377868 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 18 22:28:34.377868 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 18 22:28:34.377868 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:28:34.377868 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 22:28:35.769795 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=16 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:28:35.917686 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=32,nova_api:UPDATE=2,nova_api:DELETE=2 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:28:36.064151 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:28:36.064881 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell1:SELECT=22,nova_cell1:SAVEPOINT=1,nova_cell1:INSERT=35,nova_cell1:RELEASE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:28:44.167482 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=23,nova_api:DELETE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:28:44.259479 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=25,nova_api:UPDATE=2,nova_api:DELETE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:28:44.376139 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=8 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:28:44.384109 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell1:SELECT=24,nova_cell1:INSERT=37,nova_cell1:SAVEPOINT=1,nova_cell1:RELEASE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:28:52.154372 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-13793894-2cd7-4a99-8733-1a36efe46cde tempest-TaggedBootDevicesTest_v242-795566108 tempest-TaggedBootDevicesTest_v242-795566108-project] Took 0.84 seconds to select destinations for 1 instance(s). {{(pid=120658) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 18 22:28:52.176012 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-13793894-2cd7-4a99-8733-1a36efe46cde tempest-TaggedBootDevicesTest_v242-795566108 tempest-TaggedBootDevicesTest_v242-795566108-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:28:52.176476 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-13793894-2cd7-4a99-8733-1a36efe46cde tempest-TaggedBootDevicesTest_v242-795566108 tempest-TaggedBootDevicesTest_v242-795566108-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:28:52.176870 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-13793894-2cd7-4a99-8733-1a36efe46cde tempest-TaggedBootDevicesTest_v242-795566108 tempest-TaggedBootDevicesTest_v242-795566108-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:28:52.407404 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=20,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:28:52.410548 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-13793894-2cd7-4a99-8733-1a36efe46cde tempest-TaggedBootDevicesTest_v242-795566108 tempest-TaggedBootDevicesTest_v242-795566108-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:28:52.410978 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-13793894-2cd7-4a99-8733-1a36efe46cde tempest-TaggedBootDevicesTest_v242-795566108 tempest-TaggedBootDevicesTest_v242-795566108-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:28:52.411366 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-13793894-2cd7-4a99-8733-1a36efe46cde tempest-TaggedBootDevicesTest_v242-795566108 tempest-TaggedBootDevicesTest_v242-795566108-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:28:52.412053 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-13793894-2cd7-4a99-8733-1a36efe46cde tempest-TaggedBootDevicesTest_v242-795566108 tempest-TaggedBootDevicesTest_v242-795566108-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:28:52.412462 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-13793894-2cd7-4a99-8733-1a36efe46cde tempest-TaggedBootDevicesTest_v242-795566108 tempest-TaggedBootDevicesTest_v242-795566108-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:28:52.412839 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-13793894-2cd7-4a99-8733-1a36efe46cde tempest-TaggedBootDevicesTest_v242-795566108 tempest-TaggedBootDevicesTest_v242-795566108-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:28:52.432889 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-13793894-2cd7-4a99-8733-1a36efe46cde tempest-TaggedBootDevicesTest_v242-795566108 tempest-TaggedBootDevicesTest_v242-795566108-project] Getting quotas for project 8dd733c3dcfb4a1fbe56f300c8276354. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 18 22:28:52.437786 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-13793894-2cd7-4a99-8733-1a36efe46cde tempest-TaggedBootDevicesTest_v242-795566108 tempest-TaggedBootDevicesTest_v242-795566108-project] Getting quotas for user ec9a4d5f0b12408f8b3a31637e43af74 and project 8dd733c3dcfb4a1fbe56f300c8276354. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 18 22:28:52.448510 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-13793894-2cd7-4a99-8733-1a36efe46cde tempest-TaggedBootDevicesTest_v242-795566108 tempest-TaggedBootDevicesTest_v242-795566108-project] [instance: e7c2923e-b57f-448e-93a1-e9288e28fed8] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120658) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 18 22:28:52.449283 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-13793894-2cd7-4a99-8733-1a36efe46cde tempest-TaggedBootDevicesTest_v242-795566108 tempest-TaggedBootDevicesTest_v242-795566108-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:28:52.449749 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-13793894-2cd7-4a99-8733-1a36efe46cde tempest-TaggedBootDevicesTest_v242-795566108 tempest-TaggedBootDevicesTest_v242-795566108-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:28:52.450244 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-13793894-2cd7-4a99-8733-1a36efe46cde tempest-TaggedBootDevicesTest_v242-795566108 tempest-TaggedBootDevicesTest_v242-795566108-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:28:52.509096 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-13793894-2cd7-4a99-8733-1a36efe46cde tempest-TaggedBootDevicesTest_v242-795566108 tempest-TaggedBootDevicesTest_v242-795566108-project] [instance: e7c2923e-b57f-448e-93a1-e9288e28fed8] block_device_mapping [BlockDeviceMapping(attachment_id=8223cc8e-481d-45cf-8b4d-b0d9ac0fee60,boot_index=0,connection_info=None,created_at=,delete_on_termination=False,deleted=,deleted_at=,destination_type='volume',device_name=None,device_type=None,disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id=None,instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='volume',tag='boot',updated_at=,uuid=,volume_id='94f2421e-8387-46f0-b8b7-03a71ad84cd7',volume_size=1,volume_type=None), BlockDeviceMapping(attachment_id=73633838-d1d7-4341-8162-532b6f9013b1,boot_index=1,connection_info=None,created_at=,delete_on_termination=False,deleted=,deleted_at=,destination_type='volume',device_name=None,device_type=None,disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id=None,instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='volume',tag='other',updated_at=,uuid=,volume_id='d8b2b3ed-2b8d-4e2c-a162-746ca123f558',volume_size=1,volume_type=None), BlockDeviceMapping(attachment_id=1d69e31d-9a27-4bf7-b459-dc39b642d00a,boot_index=2,connection_info=None,created_at=,delete_on_termination=False,deleted=,deleted_at=,destination_type='volume',device_name=None,device_type=None,disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id=None,instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='volume',tag=None,updated_at=,uuid=,volume_id='f681e892-73c7-4210-b723-c0dc790c713d',volume_size=1,volume_type=None)] {{(pid=120658) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 18 22:28:52.510934 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-13793894-2cd7-4a99-8733-1a36efe46cde tempest-TaggedBootDevicesTest_v242-795566108 tempest-TaggedBootDevicesTest_v242-795566108-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:28:52.511602 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-13793894-2cd7-4a99-8733-1a36efe46cde tempest-TaggedBootDevicesTest_v242-795566108 tempest-TaggedBootDevicesTest_v242-795566108-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:28:52.512020 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-13793894-2cd7-4a99-8733-1a36efe46cde tempest-TaggedBootDevicesTest_v242-795566108 tempest-TaggedBootDevicesTest_v242-795566108-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:28:52.611604 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-13793894-2cd7-4a99-8733-1a36efe46cde tempest-TaggedBootDevicesTest_v242-795566108 tempest-TaggedBootDevicesTest_v242-795566108-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:28:52.612072 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-13793894-2cd7-4a99-8733-1a36efe46cde tempest-TaggedBootDevicesTest_v242-795566108 tempest-TaggedBootDevicesTest_v242-795566108-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:28:52.612459 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-13793894-2cd7-4a99-8733-1a36efe46cde tempest-TaggedBootDevicesTest_v242-795566108 tempest-TaggedBootDevicesTest_v242-795566108-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:28:52.709330 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-13793894-2cd7-4a99-8733-1a36efe46cde tempest-TaggedBootDevicesTest_v242-795566108 tempest-TaggedBootDevicesTest_v242-795566108-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:28:52.709697 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-13793894-2cd7-4a99-8733-1a36efe46cde tempest-TaggedBootDevicesTest_v242-795566108 tempest-TaggedBootDevicesTest_v242-795566108-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:28:52.710085 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-13793894-2cd7-4a99-8733-1a36efe46cde tempest-TaggedBootDevicesTest_v242-795566108 tempest-TaggedBootDevicesTest_v242-795566108-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:28:52.949158 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-13793894-2cd7-4a99-8733-1a36efe46cde tempest-TaggedBootDevicesTest_v242-795566108 tempest-TaggedBootDevicesTest_v242-795566108-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:28:52.949609 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-13793894-2cd7-4a99-8733-1a36efe46cde tempest-TaggedBootDevicesTest_v242-795566108 tempest-TaggedBootDevicesTest_v242-795566108-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:28:52.950529 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-13793894-2cd7-4a99-8733-1a36efe46cde tempest-TaggedBootDevicesTest_v242-795566108 tempest-TaggedBootDevicesTest_v242-795566108-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:28:52.951156 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-13793894-2cd7-4a99-8733-1a36efe46cde tempest-TaggedBootDevicesTest_v242-795566108 tempest-TaggedBootDevicesTest_v242-795566108-project] Acquiring lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 18 22:28:52.951505 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-13793894-2cd7-4a99-8733-1a36efe46cde tempest-TaggedBootDevicesTest_v242-795566108 tempest-TaggedBootDevicesTest_v242-795566108-project] Acquired lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 18 22:28:52.952373 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-19dc7531-aa41-4665-b894-e35cbf876427 None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:28:52.952790 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-19dc7531-aa41-4665-b894-e35cbf876427 None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:28:52.953193 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-19dc7531-aa41-4665-b894-e35cbf876427 None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:28:52.953837 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-19dc7531-aa41-4665-b894-e35cbf876427 None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:28:52.954242 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-19dc7531-aa41-4665-b894-e35cbf876427 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:28:52.954637 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-19dc7531-aa41-4665-b894-e35cbf876427 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:28:52.963011 n-d-853704-1 nova-conductor[120658]: ERROR nova.compute.rpcapi [None req-19dc7531-aa41-4665-b894-e35cbf876427 None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 18 22:28:52.963490 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-19dc7531-aa41-4665-b894-e35cbf876427 None None] Releasing lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 18 22:28:52.964383 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server [None req-19dc7531-aa41-4665-b894-e35cbf876427 None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:28:52.964383 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:28:52.964383 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 18 22:28:52.964383 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 18 22:28:52.964383 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 18 22:28:52.964383 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 22:28:52.964383 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 18 22:28:52.964383 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 22:28:52.964383 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:28:52.964383 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 18 22:28:52.964383 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 18 22:28:52.964383 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 18 22:28:52.964383 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 18 22:28:52.964383 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 18 22:28:52.964383 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 18 22:28:52.964383 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 18 22:28:52.964383 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 18 22:28:52.964383 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 18 22:28:52.964383 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 18 22:28:52.964383 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 18 22:28:52.964383 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 18 22:28:52.964383 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 18 22:28:52.964383 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 18 22:28:52.964383 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:28:52.964383 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 22:29:02.804236 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=13,nova_api:UPDATE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:29:02.866304 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=11,nova_api:DELETE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:29:02.961274 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:29:02.962154 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell1:SELECT=13,nova_cell1:SAVEPOINT=1,nova_cell1:INSERT=20,nova_cell1:RELEASE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:29:03.102394 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=19,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:29:03.133343 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=7,nova_cell0:UPDATE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:29:33.115197 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=6,nova_cell0:UPDATE=3 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:29:43.147572 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:30:03.115909 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=6,nova_cell0:UPDATE=3 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:30:23.151092 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:30:43.120343 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:30:53.152535 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=5,nova_cell0:UPDATE=3 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:31:10.430567 n-d-853704-1 nova-conductor[120657]: ERROR oslo.messaging._drivers.impl_rabbit [None req-c29a4046-6763-44b6-a15c-10289f713258 tempest-ServersTestFqdnHostnames-1706151809 tempest-ServersTestFqdnHostnames-1706151809-project] [ae71ad44-1ef2-45ec-82d0-4ccfc1149604] AMQP server on 10.222.0.49:5672 is unreachable: [Errno 104] Connection reset by peer. Trying again in 1 seconds.: ConnectionResetError: [Errno 104] Connection reset by peer Aug 18 22:31:11.438162 n-d-853704-1 nova-conductor[120657]: INFO oslo.messaging._drivers.impl_rabbit [None req-c29a4046-6763-44b6-a15c-10289f713258 tempest-ServersTestFqdnHostnames-1706151809 tempest-ServersTestFqdnHostnames-1706151809-project] [ae71ad44-1ef2-45ec-82d0-4ccfc1149604] Reconnected to AMQP server on 10.222.0.49:5672 via [amqp] client with port 48648. Aug 18 22:31:14.197079 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-c29a4046-6763-44b6-a15c-10289f713258 tempest-ServersTestFqdnHostnames-1706151809 tempest-ServersTestFqdnHostnames-1706151809-project] Took 3.77 seconds to select destinations for 1 instance(s). {{(pid=120657) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 18 22:31:14.218405 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c29a4046-6763-44b6-a15c-10289f713258 tempest-ServersTestFqdnHostnames-1706151809 tempest-ServersTestFqdnHostnames-1706151809-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:31:14.218841 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c29a4046-6763-44b6-a15c-10289f713258 tempest-ServersTestFqdnHostnames-1706151809 tempest-ServersTestFqdnHostnames-1706151809-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:31:14.219230 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c29a4046-6763-44b6-a15c-10289f713258 tempest-ServersTestFqdnHostnames-1706151809 tempest-ServersTestFqdnHostnames-1706151809-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:31:14.370568 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:31:14.374415 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c29a4046-6763-44b6-a15c-10289f713258 tempest-ServersTestFqdnHostnames-1706151809 tempest-ServersTestFqdnHostnames-1706151809-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:31:14.374814 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c29a4046-6763-44b6-a15c-10289f713258 tempest-ServersTestFqdnHostnames-1706151809 tempest-ServersTestFqdnHostnames-1706151809-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:31:14.375215 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c29a4046-6763-44b6-a15c-10289f713258 tempest-ServersTestFqdnHostnames-1706151809 tempest-ServersTestFqdnHostnames-1706151809-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:31:14.375906 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c29a4046-6763-44b6-a15c-10289f713258 tempest-ServersTestFqdnHostnames-1706151809 tempest-ServersTestFqdnHostnames-1706151809-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:31:14.376364 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c29a4046-6763-44b6-a15c-10289f713258 tempest-ServersTestFqdnHostnames-1706151809 tempest-ServersTestFqdnHostnames-1706151809-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:31:14.376731 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c29a4046-6763-44b6-a15c-10289f713258 tempest-ServersTestFqdnHostnames-1706151809 tempest-ServersTestFqdnHostnames-1706151809-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:31:14.393432 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-c29a4046-6763-44b6-a15c-10289f713258 tempest-ServersTestFqdnHostnames-1706151809 tempest-ServersTestFqdnHostnames-1706151809-project] Getting quotas for project ad86a0f4423a488ab65d485198e12dc1. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 18 22:31:14.397560 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-c29a4046-6763-44b6-a15c-10289f713258 tempest-ServersTestFqdnHostnames-1706151809 tempest-ServersTestFqdnHostnames-1706151809-project] Getting quotas for user a2801d7b6c974e82bfe42c53d72a8d8d and project ad86a0f4423a488ab65d485198e12dc1. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 18 22:31:14.409307 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-c29a4046-6763-44b6-a15c-10289f713258 tempest-ServersTestFqdnHostnames-1706151809 tempest-ServersTestFqdnHostnames-1706151809-project] [instance: 05c6f8fb-9ba3-4f3e-a36a-018709fb5e4f] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120657) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 18 22:31:14.410079 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c29a4046-6763-44b6-a15c-10289f713258 tempest-ServersTestFqdnHostnames-1706151809 tempest-ServersTestFqdnHostnames-1706151809-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:31:14.410508 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c29a4046-6763-44b6-a15c-10289f713258 tempest-ServersTestFqdnHostnames-1706151809 tempest-ServersTestFqdnHostnames-1706151809-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:31:14.410892 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c29a4046-6763-44b6-a15c-10289f713258 tempest-ServersTestFqdnHostnames-1706151809 tempest-ServersTestFqdnHostnames-1706151809-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:31:14.492256 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-c29a4046-6763-44b6-a15c-10289f713258 tempest-ServersTestFqdnHostnames-1706151809 tempest-ServersTestFqdnHostnames-1706151809-project] [instance: 05c6f8fb-9ba3-4f3e-a36a-018709fb5e4f] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='87b0fec9-a948-41f9-a3c0-f0a6a5a8f1e7',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120657) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 18 22:31:14.493465 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c29a4046-6763-44b6-a15c-10289f713258 tempest-ServersTestFqdnHostnames-1706151809 tempest-ServersTestFqdnHostnames-1706151809-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:31:14.493905 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c29a4046-6763-44b6-a15c-10289f713258 tempest-ServersTestFqdnHostnames-1706151809 tempest-ServersTestFqdnHostnames-1706151809-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:31:14.494299 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c29a4046-6763-44b6-a15c-10289f713258 tempest-ServersTestFqdnHostnames-1706151809 tempest-ServersTestFqdnHostnames-1706151809-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:31:14.750350 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c29a4046-6763-44b6-a15c-10289f713258 tempest-ServersTestFqdnHostnames-1706151809 tempest-ServersTestFqdnHostnames-1706151809-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:31:14.750797 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c29a4046-6763-44b6-a15c-10289f713258 tempest-ServersTestFqdnHostnames-1706151809 tempest-ServersTestFqdnHostnames-1706151809-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:31:14.751194 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c29a4046-6763-44b6-a15c-10289f713258 tempest-ServersTestFqdnHostnames-1706151809 tempest-ServersTestFqdnHostnames-1706151809-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:31:14.751647 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c29a4046-6763-44b6-a15c-10289f713258 tempest-ServersTestFqdnHostnames-1706151809 tempest-ServersTestFqdnHostnames-1706151809-project] Acquiring lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 18 22:31:14.752030 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c29a4046-6763-44b6-a15c-10289f713258 tempest-ServersTestFqdnHostnames-1706151809 tempest-ServersTestFqdnHostnames-1706151809-project] Acquired lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 18 22:31:14.752797 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-972b964f-96ad-49e7-9916-4407a5771877 None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:31:14.753234 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-972b964f-96ad-49e7-9916-4407a5771877 None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:31:14.753638 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-972b964f-96ad-49e7-9916-4407a5771877 None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:31:14.754283 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-972b964f-96ad-49e7-9916-4407a5771877 None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:31:14.754697 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-972b964f-96ad-49e7-9916-4407a5771877 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:31:14.755067 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-972b964f-96ad-49e7-9916-4407a5771877 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:31:14.764931 n-d-853704-1 nova-conductor[120657]: ERROR nova.compute.rpcapi [None req-972b964f-96ad-49e7-9916-4407a5771877 None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 18 22:31:14.765322 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-972b964f-96ad-49e7-9916-4407a5771877 None None] Releasing lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 18 22:31:14.766222 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server [None req-972b964f-96ad-49e7-9916-4407a5771877 None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:31:14.766222 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:31:14.766222 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 18 22:31:14.766222 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 18 22:31:14.766222 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 18 22:31:14.766222 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 22:31:14.766222 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 18 22:31:14.766222 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 22:31:14.766222 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:31:14.766222 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 18 22:31:14.766222 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 18 22:31:14.766222 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 18 22:31:14.766222 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 18 22:31:14.766222 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 18 22:31:14.766222 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 18 22:31:14.766222 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 18 22:31:14.766222 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 18 22:31:14.766222 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 18 22:31:14.766222 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 18 22:31:14.766222 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 18 22:31:14.766222 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 18 22:31:14.766222 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 18 22:31:14.766222 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 18 22:31:14.766222 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:31:14.766222 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 22:31:24.666210 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=11 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:31:24.667169 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=13,nova_api:UPDATE=1,nova_api:DELETE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:31:24.761972 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:31:24.762813 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell1:SELECT=11,nova_cell1:SAVEPOINT=1,nova_cell1:INSERT=18,nova_cell1:RELEASE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:31:33.143021 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:31:41.252338 n-d-853704-1 nova-conductor[120658]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 18 22:31:53.127505 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=11,nova_cell0:UPDATE=3 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:32:04.320603 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-cbeba05a-3c8a-45bd-9701-c50da0fb0618 tempest-ServerRescueTestJSON-2003987351 tempest-ServerRescueTestJSON-2003987351-project] Took 0.61 seconds to select destinations for 1 instance(s). {{(pid=120658) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 18 22:32:04.364565 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-cbeba05a-3c8a-45bd-9701-c50da0fb0618 tempest-ServerRescueTestJSON-2003987351 tempest-ServerRescueTestJSON-2003987351-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:32:04.365123 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-cbeba05a-3c8a-45bd-9701-c50da0fb0618 tempest-ServerRescueTestJSON-2003987351 tempest-ServerRescueTestJSON-2003987351-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:32:04.365616 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-cbeba05a-3c8a-45bd-9701-c50da0fb0618 tempest-ServerRescueTestJSON-2003987351 tempest-ServerRescueTestJSON-2003987351-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:32:04.552131 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:32:04.554150 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-cbeba05a-3c8a-45bd-9701-c50da0fb0618 tempest-ServerRescueTestJSON-2003987351 tempest-ServerRescueTestJSON-2003987351-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:32:04.554587 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-cbeba05a-3c8a-45bd-9701-c50da0fb0618 tempest-ServerRescueTestJSON-2003987351 tempest-ServerRescueTestJSON-2003987351-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:32:04.554973 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-cbeba05a-3c8a-45bd-9701-c50da0fb0618 tempest-ServerRescueTestJSON-2003987351 tempest-ServerRescueTestJSON-2003987351-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:32:04.555658 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-cbeba05a-3c8a-45bd-9701-c50da0fb0618 tempest-ServerRescueTestJSON-2003987351 tempest-ServerRescueTestJSON-2003987351-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:32:04.556100 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-cbeba05a-3c8a-45bd-9701-c50da0fb0618 tempest-ServerRescueTestJSON-2003987351 tempest-ServerRescueTestJSON-2003987351-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:32:04.558293 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-cbeba05a-3c8a-45bd-9701-c50da0fb0618 tempest-ServerRescueTestJSON-2003987351 tempest-ServerRescueTestJSON-2003987351-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:32:04.571358 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-cbeba05a-3c8a-45bd-9701-c50da0fb0618 tempest-ServerRescueTestJSON-2003987351 tempest-ServerRescueTestJSON-2003987351-project] Getting quotas for project f5696ab3f8af43f1b6bebf412038a550. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 18 22:32:04.575367 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-cbeba05a-3c8a-45bd-9701-c50da0fb0618 tempest-ServerRescueTestJSON-2003987351 tempest-ServerRescueTestJSON-2003987351-project] Getting quotas for user d0c61bfa6c4a430c96f6071851c78ec2 and project f5696ab3f8af43f1b6bebf412038a550. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 18 22:32:04.585825 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-cbeba05a-3c8a-45bd-9701-c50da0fb0618 tempest-ServerRescueTestJSON-2003987351 tempest-ServerRescueTestJSON-2003987351-project] [instance: 9db70e54-0a5b-4baf-8cf4-e3a30b2d0e3b] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120658) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 18 22:32:04.586589 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-cbeba05a-3c8a-45bd-9701-c50da0fb0618 tempest-ServerRescueTestJSON-2003987351 tempest-ServerRescueTestJSON-2003987351-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:32:04.587022 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-cbeba05a-3c8a-45bd-9701-c50da0fb0618 tempest-ServerRescueTestJSON-2003987351 tempest-ServerRescueTestJSON-2003987351-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:32:04.587403 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-cbeba05a-3c8a-45bd-9701-c50da0fb0618 tempest-ServerRescueTestJSON-2003987351 tempest-ServerRescueTestJSON-2003987351-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:32:04.662191 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-cbeba05a-3c8a-45bd-9701-c50da0fb0618 tempest-ServerRescueTestJSON-2003987351 tempest-ServerRescueTestJSON-2003987351-project] [instance: 9db70e54-0a5b-4baf-8cf4-e3a30b2d0e3b] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='87b0fec9-a948-41f9-a3c0-f0a6a5a8f1e7',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120658) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 18 22:32:04.663407 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-cbeba05a-3c8a-45bd-9701-c50da0fb0618 tempest-ServerRescueTestJSON-2003987351 tempest-ServerRescueTestJSON-2003987351-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:32:04.663887 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-cbeba05a-3c8a-45bd-9701-c50da0fb0618 tempest-ServerRescueTestJSON-2003987351 tempest-ServerRescueTestJSON-2003987351-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:32:04.664288 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-cbeba05a-3c8a-45bd-9701-c50da0fb0618 tempest-ServerRescueTestJSON-2003987351 tempest-ServerRescueTestJSON-2003987351-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:32:04.883411 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-cbeba05a-3c8a-45bd-9701-c50da0fb0618 tempest-ServerRescueTestJSON-2003987351 tempest-ServerRescueTestJSON-2003987351-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:32:04.883873 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-cbeba05a-3c8a-45bd-9701-c50da0fb0618 tempest-ServerRescueTestJSON-2003987351 tempest-ServerRescueTestJSON-2003987351-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:32:04.884290 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-cbeba05a-3c8a-45bd-9701-c50da0fb0618 tempest-ServerRescueTestJSON-2003987351 tempest-ServerRescueTestJSON-2003987351-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:32:04.884744 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-cbeba05a-3c8a-45bd-9701-c50da0fb0618 tempest-ServerRescueTestJSON-2003987351 tempest-ServerRescueTestJSON-2003987351-project] Acquiring lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 18 22:32:04.885098 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-cbeba05a-3c8a-45bd-9701-c50da0fb0618 tempest-ServerRescueTestJSON-2003987351 tempest-ServerRescueTestJSON-2003987351-project] Acquired lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 18 22:32:04.885832 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-be5bbc2b-265e-4029-9229-b4025224e92a None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:32:04.886274 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-be5bbc2b-265e-4029-9229-b4025224e92a None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:32:04.886656 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-be5bbc2b-265e-4029-9229-b4025224e92a None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:32:04.887334 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-be5bbc2b-265e-4029-9229-b4025224e92a None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:32:04.888514 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-be5bbc2b-265e-4029-9229-b4025224e92a None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:32:04.889083 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-be5bbc2b-265e-4029-9229-b4025224e92a None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:32:04.898260 n-d-853704-1 nova-conductor[120658]: ERROR nova.compute.rpcapi [None req-be5bbc2b-265e-4029-9229-b4025224e92a None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 18 22:32:04.898651 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-be5bbc2b-265e-4029-9229-b4025224e92a None None] Releasing lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 18 22:32:04.899558 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server [None req-be5bbc2b-265e-4029-9229-b4025224e92a None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:32:04.899558 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:32:04.899558 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 18 22:32:04.899558 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 18 22:32:04.899558 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 18 22:32:04.899558 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 22:32:04.899558 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 18 22:32:04.899558 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 22:32:04.899558 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:32:04.899558 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 18 22:32:04.899558 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 18 22:32:04.899558 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 18 22:32:04.899558 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 18 22:32:04.899558 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 18 22:32:04.899558 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 18 22:32:04.899558 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 18 22:32:04.899558 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 18 22:32:04.899558 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 18 22:32:04.899558 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 18 22:32:04.899558 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 18 22:32:04.899558 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 18 22:32:04.899558 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 18 22:32:04.899558 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 18 22:32:04.899558 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:32:04.899558 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 22:32:14.812583 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=11 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:32:14.813545 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=13,nova_api:UPDATE=1,nova_api:DELETE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:32:14.895368 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:32:14.896151 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell1:SELECT=11,nova_cell1:SAVEPOINT=1,nova_cell1:INSERT=18,nova_cell1:RELEASE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:32:23.056951 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-db666c63-4318-4115-9dd6-043501723d0b tempest-MigrationsAdminTest-1637837745 tempest-MigrationsAdminTest-1637837745-project] Took 0.40 seconds to select destinations for 1 instance(s). {{(pid=120657) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 18 22:32:23.078906 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-db666c63-4318-4115-9dd6-043501723d0b tempest-MigrationsAdminTest-1637837745 tempest-MigrationsAdminTest-1637837745-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:32:23.079347 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-db666c63-4318-4115-9dd6-043501723d0b tempest-MigrationsAdminTest-1637837745 tempest-MigrationsAdminTest-1637837745-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:32:23.079773 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-db666c63-4318-4115-9dd6-043501723d0b tempest-MigrationsAdminTest-1637837745 tempest-MigrationsAdminTest-1637837745-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:32:23.211514 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-db666c63-4318-4115-9dd6-043501723d0b tempest-MigrationsAdminTest-1637837745 tempest-MigrationsAdminTest-1637837745-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:32:23.211958 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-db666c63-4318-4115-9dd6-043501723d0b tempest-MigrationsAdminTest-1637837745 tempest-MigrationsAdminTest-1637837745-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:32:23.212356 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-db666c63-4318-4115-9dd6-043501723d0b tempest-MigrationsAdminTest-1637837745 tempest-MigrationsAdminTest-1637837745-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:32:23.213015 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-db666c63-4318-4115-9dd6-043501723d0b tempest-MigrationsAdminTest-1637837745 tempest-MigrationsAdminTest-1637837745-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:32:23.213421 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-db666c63-4318-4115-9dd6-043501723d0b tempest-MigrationsAdminTest-1637837745 tempest-MigrationsAdminTest-1637837745-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:32:23.213996 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-db666c63-4318-4115-9dd6-043501723d0b tempest-MigrationsAdminTest-1637837745 tempest-MigrationsAdminTest-1637837745-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:32:23.230977 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-db666c63-4318-4115-9dd6-043501723d0b tempest-MigrationsAdminTest-1637837745 tempest-MigrationsAdminTest-1637837745-project] Getting quotas for project 55cd873dadd6496dbebea5c3290775eb. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 18 22:32:23.235728 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-db666c63-4318-4115-9dd6-043501723d0b tempest-MigrationsAdminTest-1637837745 tempest-MigrationsAdminTest-1637837745-project] Getting quotas for user caa4813c18704375b16dedceb822364f and project 55cd873dadd6496dbebea5c3290775eb. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 18 22:32:23.246678 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-db666c63-4318-4115-9dd6-043501723d0b tempest-MigrationsAdminTest-1637837745 tempest-MigrationsAdminTest-1637837745-project] [instance: f8e087c9-d52e-433d-bc61-a205850d940b] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120657) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 18 22:32:23.247448 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-db666c63-4318-4115-9dd6-043501723d0b tempest-MigrationsAdminTest-1637837745 tempest-MigrationsAdminTest-1637837745-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:32:23.247911 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-db666c63-4318-4115-9dd6-043501723d0b tempest-MigrationsAdminTest-1637837745 tempest-MigrationsAdminTest-1637837745-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:32:23.248887 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-db666c63-4318-4115-9dd6-043501723d0b tempest-MigrationsAdminTest-1637837745 tempest-MigrationsAdminTest-1637837745-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:32:23.321411 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-db666c63-4318-4115-9dd6-043501723d0b tempest-MigrationsAdminTest-1637837745 tempest-MigrationsAdminTest-1637837745-project] [instance: f8e087c9-d52e-433d-bc61-a205850d940b] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='87b0fec9-a948-41f9-a3c0-f0a6a5a8f1e7',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120657) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 18 22:32:23.322684 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-db666c63-4318-4115-9dd6-043501723d0b tempest-MigrationsAdminTest-1637837745 tempest-MigrationsAdminTest-1637837745-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:32:23.323357 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-db666c63-4318-4115-9dd6-043501723d0b tempest-MigrationsAdminTest-1637837745 tempest-MigrationsAdminTest-1637837745-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:32:23.323804 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-db666c63-4318-4115-9dd6-043501723d0b tempest-MigrationsAdminTest-1637837745 tempest-MigrationsAdminTest-1637837745-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:32:23.553792 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-db666c63-4318-4115-9dd6-043501723d0b tempest-MigrationsAdminTest-1637837745 tempest-MigrationsAdminTest-1637837745-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:32:23.554282 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-db666c63-4318-4115-9dd6-043501723d0b tempest-MigrationsAdminTest-1637837745 tempest-MigrationsAdminTest-1637837745-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:32:23.554668 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-db666c63-4318-4115-9dd6-043501723d0b tempest-MigrationsAdminTest-1637837745 tempest-MigrationsAdminTest-1637837745-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:32:23.555104 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-db666c63-4318-4115-9dd6-043501723d0b tempest-MigrationsAdminTest-1637837745 tempest-MigrationsAdminTest-1637837745-project] Acquiring lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 18 22:32:23.555448 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-db666c63-4318-4115-9dd6-043501723d0b tempest-MigrationsAdminTest-1637837745 tempest-MigrationsAdminTest-1637837745-project] Acquired lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 18 22:32:23.556203 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-058a265b-be28-4769-91e8-dcfb4ec87192 None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:32:23.556613 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-058a265b-be28-4769-91e8-dcfb4ec87192 None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:32:23.556988 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-058a265b-be28-4769-91e8-dcfb4ec87192 None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:32:23.557618 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-058a265b-be28-4769-91e8-dcfb4ec87192 None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:32:23.558019 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-058a265b-be28-4769-91e8-dcfb4ec87192 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:32:23.558389 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-058a265b-be28-4769-91e8-dcfb4ec87192 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:32:23.567220 n-d-853704-1 nova-conductor[120657]: ERROR nova.compute.rpcapi [None req-058a265b-be28-4769-91e8-dcfb4ec87192 None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 18 22:32:23.567600 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-058a265b-be28-4769-91e8-dcfb4ec87192 None None] Releasing lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 18 22:32:23.568753 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server [None req-058a265b-be28-4769-91e8-dcfb4ec87192 None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:32:23.568753 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:32:23.568753 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 18 22:32:23.568753 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 18 22:32:23.568753 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 18 22:32:23.568753 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 22:32:23.568753 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 18 22:32:23.568753 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 22:32:23.568753 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:32:23.568753 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 18 22:32:23.568753 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 18 22:32:23.568753 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 18 22:32:23.568753 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 18 22:32:23.568753 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 18 22:32:23.568753 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 18 22:32:23.568753 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 18 22:32:23.568753 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 18 22:32:23.568753 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 18 22:32:23.568753 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 18 22:32:23.568753 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 18 22:32:23.568753 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 18 22:32:23.568753 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 18 22:32:23.568753 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 18 22:32:23.568753 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:32:23.568753 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 22:32:33.117575 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=15,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:32:33.133637 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=2 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:32:33.466108 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=14,nova_api:UPDATE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:32:33.466785 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=10,nova_api:DELETE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:32:33.565017 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:32:33.565655 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell1:SELECT=11,nova_cell1:INSERT=17 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:33:13.126895 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=7,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:33:13.135173 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:33:33.140668 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=4,nova_cell0:UPDATE=2 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:33:49.563231 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-ef79f310-869c-4abb-952a-dfe125dcdc1d tempest-AttachVolumeShelveTestJSON-783903826 tempest-AttachVolumeShelveTestJSON-783903826-project] Took 0.41 seconds to select destinations for 1 instance(s). {{(pid=120658) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 18 22:33:49.589281 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ef79f310-869c-4abb-952a-dfe125dcdc1d tempest-AttachVolumeShelveTestJSON-783903826 tempest-AttachVolumeShelveTestJSON-783903826-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:33:49.589724 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ef79f310-869c-4abb-952a-dfe125dcdc1d tempest-AttachVolumeShelveTestJSON-783903826 tempest-AttachVolumeShelveTestJSON-783903826-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:33:49.590116 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ef79f310-869c-4abb-952a-dfe125dcdc1d tempest-AttachVolumeShelveTestJSON-783903826 tempest-AttachVolumeShelveTestJSON-783903826-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:33:49.736990 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ef79f310-869c-4abb-952a-dfe125dcdc1d tempest-AttachVolumeShelveTestJSON-783903826 tempest-AttachVolumeShelveTestJSON-783903826-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:33:49.737434 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ef79f310-869c-4abb-952a-dfe125dcdc1d tempest-AttachVolumeShelveTestJSON-783903826 tempest-AttachVolumeShelveTestJSON-783903826-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:33:49.737821 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ef79f310-869c-4abb-952a-dfe125dcdc1d tempest-AttachVolumeShelveTestJSON-783903826 tempest-AttachVolumeShelveTestJSON-783903826-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:33:49.738477 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ef79f310-869c-4abb-952a-dfe125dcdc1d tempest-AttachVolumeShelveTestJSON-783903826 tempest-AttachVolumeShelveTestJSON-783903826-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:33:49.738888 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ef79f310-869c-4abb-952a-dfe125dcdc1d tempest-AttachVolumeShelveTestJSON-783903826 tempest-AttachVolumeShelveTestJSON-783903826-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:33:49.739261 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ef79f310-869c-4abb-952a-dfe125dcdc1d tempest-AttachVolumeShelveTestJSON-783903826 tempest-AttachVolumeShelveTestJSON-783903826-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:33:49.755934 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-ef79f310-869c-4abb-952a-dfe125dcdc1d tempest-AttachVolumeShelveTestJSON-783903826 tempest-AttachVolumeShelveTestJSON-783903826-project] Getting quotas for project 9490452f4e584b88a243d315381fa891. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 18 22:33:49.760142 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-ef79f310-869c-4abb-952a-dfe125dcdc1d tempest-AttachVolumeShelveTestJSON-783903826 tempest-AttachVolumeShelveTestJSON-783903826-project] Getting quotas for user b9b86470f04244d3b60ab4b3d42f3539 and project 9490452f4e584b88a243d315381fa891. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 18 22:33:49.771225 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-ef79f310-869c-4abb-952a-dfe125dcdc1d tempest-AttachVolumeShelveTestJSON-783903826 tempest-AttachVolumeShelveTestJSON-783903826-project] [instance: f1a5507c-0128-4781-92d9-987b435a3ebf] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120658) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 18 22:33:49.772031 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ef79f310-869c-4abb-952a-dfe125dcdc1d tempest-AttachVolumeShelveTestJSON-783903826 tempest-AttachVolumeShelveTestJSON-783903826-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:33:49.772470 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ef79f310-869c-4abb-952a-dfe125dcdc1d tempest-AttachVolumeShelveTestJSON-783903826 tempest-AttachVolumeShelveTestJSON-783903826-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:33:49.772859 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ef79f310-869c-4abb-952a-dfe125dcdc1d tempest-AttachVolumeShelveTestJSON-783903826 tempest-AttachVolumeShelveTestJSON-783903826-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:33:49.832627 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-ef79f310-869c-4abb-952a-dfe125dcdc1d tempest-AttachVolumeShelveTestJSON-783903826 tempest-AttachVolumeShelveTestJSON-783903826-project] [instance: f1a5507c-0128-4781-92d9-987b435a3ebf] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='87b0fec9-a948-41f9-a3c0-f0a6a5a8f1e7',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120658) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 18 22:33:49.837137 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ef79f310-869c-4abb-952a-dfe125dcdc1d tempest-AttachVolumeShelveTestJSON-783903826 tempest-AttachVolumeShelveTestJSON-783903826-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:33:49.837137 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ef79f310-869c-4abb-952a-dfe125dcdc1d tempest-AttachVolumeShelveTestJSON-783903826 tempest-AttachVolumeShelveTestJSON-783903826-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:33:49.837137 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ef79f310-869c-4abb-952a-dfe125dcdc1d tempest-AttachVolumeShelveTestJSON-783903826 tempest-AttachVolumeShelveTestJSON-783903826-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:33:50.033355 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ef79f310-869c-4abb-952a-dfe125dcdc1d tempest-AttachVolumeShelveTestJSON-783903826 tempest-AttachVolumeShelveTestJSON-783903826-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:33:50.033792 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ef79f310-869c-4abb-952a-dfe125dcdc1d tempest-AttachVolumeShelveTestJSON-783903826 tempest-AttachVolumeShelveTestJSON-783903826-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:33:50.034187 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ef79f310-869c-4abb-952a-dfe125dcdc1d tempest-AttachVolumeShelveTestJSON-783903826 tempest-AttachVolumeShelveTestJSON-783903826-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:33:50.034638 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ef79f310-869c-4abb-952a-dfe125dcdc1d tempest-AttachVolumeShelveTestJSON-783903826 tempest-AttachVolumeShelveTestJSON-783903826-project] Acquiring lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 18 22:33:50.034985 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ef79f310-869c-4abb-952a-dfe125dcdc1d tempest-AttachVolumeShelveTestJSON-783903826 tempest-AttachVolumeShelveTestJSON-783903826-project] Acquired lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 18 22:33:50.035778 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-375457ca-55d6-4d0a-ae03-8f79d8f0c791 None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:33:50.036189 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-375457ca-55d6-4d0a-ae03-8f79d8f0c791 None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:33:50.036571 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-375457ca-55d6-4d0a-ae03-8f79d8f0c791 None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:33:50.037312 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-375457ca-55d6-4d0a-ae03-8f79d8f0c791 None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:33:50.037777 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-375457ca-55d6-4d0a-ae03-8f79d8f0c791 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:33:50.038099 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-375457ca-55d6-4d0a-ae03-8f79d8f0c791 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:33:50.046870 n-d-853704-1 nova-conductor[120658]: ERROR nova.compute.rpcapi [None req-375457ca-55d6-4d0a-ae03-8f79d8f0c791 None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 18 22:33:50.047264 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-375457ca-55d6-4d0a-ae03-8f79d8f0c791 None None] Releasing lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 18 22:33:50.049448 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server [None req-375457ca-55d6-4d0a-ae03-8f79d8f0c791 None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:33:50.049448 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:33:50.049448 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 18 22:33:50.049448 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 18 22:33:50.049448 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 18 22:33:50.049448 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 22:33:50.049448 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 18 22:33:50.049448 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 22:33:50.049448 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:33:50.049448 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 18 22:33:50.049448 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 18 22:33:50.049448 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 18 22:33:50.049448 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 18 22:33:50.049448 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 18 22:33:50.049448 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 18 22:33:50.049448 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 18 22:33:50.049448 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 18 22:33:50.049448 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 18 22:33:50.049448 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 18 22:33:50.049448 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 18 22:33:50.049448 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 18 22:33:50.049448 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 18 22:33:50.049448 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 18 22:33:50.049448 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:33:50.049448 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 22:33:53.129899 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:33:59.960732 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=12,nova_api:UPDATE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:33:59.961582 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=12,nova_api:DELETE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:34:00.043477 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:34:00.044791 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell1:SELECT=11,nova_cell1:INSERT=17 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:34:13.142225 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=15,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:34:33.136368 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:34:53.147291 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:34:53.150626 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=3,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:34:59.585604 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-bfc5e7da-dcd6-4157-9dbc-607c546624a8 tempest-ServerRescueTestJSONUnderV235-1462930954 tempest-ServerRescueTestJSONUnderV235-1462930954-project] Took 0.73 seconds to select destinations for 1 instance(s). {{(pid=120657) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 18 22:34:59.616169 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-bfc5e7da-dcd6-4157-9dbc-607c546624a8 tempest-ServerRescueTestJSONUnderV235-1462930954 tempest-ServerRescueTestJSONUnderV235-1462930954-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:34:59.616837 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-bfc5e7da-dcd6-4157-9dbc-607c546624a8 tempest-ServerRescueTestJSONUnderV235-1462930954 tempest-ServerRescueTestJSONUnderV235-1462930954-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:34:59.617428 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-bfc5e7da-dcd6-4157-9dbc-607c546624a8 tempest-ServerRescueTestJSONUnderV235-1462930954 tempest-ServerRescueTestJSONUnderV235-1462930954-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:34:59.814226 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-bfc5e7da-dcd6-4157-9dbc-607c546624a8 tempest-ServerRescueTestJSONUnderV235-1462930954 tempest-ServerRescueTestJSONUnderV235-1462930954-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:34:59.827401 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-bfc5e7da-dcd6-4157-9dbc-607c546624a8 tempest-ServerRescueTestJSONUnderV235-1462930954 tempest-ServerRescueTestJSONUnderV235-1462930954-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.013s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:34:59.828793 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-bfc5e7da-dcd6-4157-9dbc-607c546624a8 tempest-ServerRescueTestJSONUnderV235-1462930954 tempest-ServerRescueTestJSONUnderV235-1462930954-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:34:59.829987 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-bfc5e7da-dcd6-4157-9dbc-607c546624a8 tempest-ServerRescueTestJSONUnderV235-1462930954 tempest-ServerRescueTestJSONUnderV235-1462930954-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:34:59.830790 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-bfc5e7da-dcd6-4157-9dbc-607c546624a8 tempest-ServerRescueTestJSONUnderV235-1462930954 tempest-ServerRescueTestJSONUnderV235-1462930954-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:34:59.831373 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-bfc5e7da-dcd6-4157-9dbc-607c546624a8 tempest-ServerRescueTestJSONUnderV235-1462930954 tempest-ServerRescueTestJSONUnderV235-1462930954-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:34:59.851075 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-bfc5e7da-dcd6-4157-9dbc-607c546624a8 tempest-ServerRescueTestJSONUnderV235-1462930954 tempest-ServerRescueTestJSONUnderV235-1462930954-project] Getting quotas for project a434a14a1758423997affe3c207f6694. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 18 22:34:59.856878 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-bfc5e7da-dcd6-4157-9dbc-607c546624a8 tempest-ServerRescueTestJSONUnderV235-1462930954 tempest-ServerRescueTestJSONUnderV235-1462930954-project] Getting quotas for user 786ef1029bd940e6946265c50dae4eba and project a434a14a1758423997affe3c207f6694. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 18 22:34:59.874659 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-bfc5e7da-dcd6-4157-9dbc-607c546624a8 tempest-ServerRescueTestJSONUnderV235-1462930954 tempest-ServerRescueTestJSONUnderV235-1462930954-project] [instance: 3a211c35-1b80-44b7-9330-fd85b5353544] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120657) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 18 22:34:59.875212 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-bfc5e7da-dcd6-4157-9dbc-607c546624a8 tempest-ServerRescueTestJSONUnderV235-1462930954 tempest-ServerRescueTestJSONUnderV235-1462930954-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:34:59.875654 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-bfc5e7da-dcd6-4157-9dbc-607c546624a8 tempest-ServerRescueTestJSONUnderV235-1462930954 tempest-ServerRescueTestJSONUnderV235-1462930954-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:34:59.876064 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-bfc5e7da-dcd6-4157-9dbc-607c546624a8 tempest-ServerRescueTestJSONUnderV235-1462930954 tempest-ServerRescueTestJSONUnderV235-1462930954-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:34:59.963629 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-bfc5e7da-dcd6-4157-9dbc-607c546624a8 tempest-ServerRescueTestJSONUnderV235-1462930954 tempest-ServerRescueTestJSONUnderV235-1462930954-project] [instance: 3a211c35-1b80-44b7-9330-fd85b5353544] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='87b0fec9-a948-41f9-a3c0-f0a6a5a8f1e7',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120657) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 18 22:34:59.966348 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-bfc5e7da-dcd6-4157-9dbc-607c546624a8 tempest-ServerRescueTestJSONUnderV235-1462930954 tempest-ServerRescueTestJSONUnderV235-1462930954-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:34:59.968601 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-bfc5e7da-dcd6-4157-9dbc-607c546624a8 tempest-ServerRescueTestJSONUnderV235-1462930954 tempest-ServerRescueTestJSONUnderV235-1462930954-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.002s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:34:59.970193 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-bfc5e7da-dcd6-4157-9dbc-607c546624a8 tempest-ServerRescueTestJSONUnderV235-1462930954 tempest-ServerRescueTestJSONUnderV235-1462930954-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.002s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:35:00.247575 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-bfc5e7da-dcd6-4157-9dbc-607c546624a8 tempest-ServerRescueTestJSONUnderV235-1462930954 tempest-ServerRescueTestJSONUnderV235-1462930954-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:35:00.248399 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-bfc5e7da-dcd6-4157-9dbc-607c546624a8 tempest-ServerRescueTestJSONUnderV235-1462930954 tempest-ServerRescueTestJSONUnderV235-1462930954-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:35:00.249271 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-bfc5e7da-dcd6-4157-9dbc-607c546624a8 tempest-ServerRescueTestJSONUnderV235-1462930954 tempest-ServerRescueTestJSONUnderV235-1462930954-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:35:00.250047 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-bfc5e7da-dcd6-4157-9dbc-607c546624a8 tempest-ServerRescueTestJSONUnderV235-1462930954 tempest-ServerRescueTestJSONUnderV235-1462930954-project] Acquiring lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 18 22:35:00.250600 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-bfc5e7da-dcd6-4157-9dbc-607c546624a8 tempest-ServerRescueTestJSONUnderV235-1462930954 tempest-ServerRescueTestJSONUnderV235-1462930954-project] Acquired lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 18 22:35:00.251637 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c2f7d4de-614f-42d5-8d9b-c2a9d9a7a653 None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:35:00.252355 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c2f7d4de-614f-42d5-8d9b-c2a9d9a7a653 None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:35:00.253118 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c2f7d4de-614f-42d5-8d9b-c2a9d9a7a653 None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:35:00.254130 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c2f7d4de-614f-42d5-8d9b-c2a9d9a7a653 None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:35:00.254781 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c2f7d4de-614f-42d5-8d9b-c2a9d9a7a653 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:35:00.255370 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c2f7d4de-614f-42d5-8d9b-c2a9d9a7a653 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:35:00.289980 n-d-853704-1 nova-conductor[120657]: ERROR nova.compute.rpcapi [None req-c2f7d4de-614f-42d5-8d9b-c2a9d9a7a653 None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 18 22:35:00.291073 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c2f7d4de-614f-42d5-8d9b-c2a9d9a7a653 None None] Releasing lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 18 22:35:00.292917 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server [None req-c2f7d4de-614f-42d5-8d9b-c2a9d9a7a653 None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:35:00.292917 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:35:00.292917 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 18 22:35:00.292917 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 18 22:35:00.292917 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 18 22:35:00.292917 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 22:35:00.292917 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 18 22:35:00.292917 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 22:35:00.292917 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:35:00.292917 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 18 22:35:00.292917 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 18 22:35:00.292917 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 18 22:35:00.292917 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 18 22:35:00.292917 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 18 22:35:00.292917 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 18 22:35:00.292917 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 18 22:35:00.292917 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 18 22:35:00.292917 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 18 22:35:00.292917 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 18 22:35:00.292917 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 18 22:35:00.292917 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 18 22:35:00.292917 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 18 22:35:00.292917 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 18 22:35:00.292917 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:35:00.292917 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 22:35:10.159558 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=8,nova_api:UPDATE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:35:10.166604 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=16,nova_api:DELETE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:35:10.285324 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:35:10.290380 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell1:SELECT=11,nova_cell1:SAVEPOINT=1,nova_cell1:INSERT=18,nova_cell1:RELEASE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:35:33.146136 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=7,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:35:33.152739 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=14,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:36:03.158446 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=6,nova_cell0:UPDATE=3 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:36:13.132915 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:36:41.262695 n-d-853704-1 nova-conductor[120658]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 18 22:36:43.161483 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:36:53.138424 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:37:03.152787 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=3,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:37:13.151246 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=3,nova_cell0:UPDATE=2 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:37:27.639399 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-2234decf-5d0d-4055-a0fa-bd58d7a762f5 tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Took 0.82 seconds to select destinations for 1 instance(s). {{(pid=120658) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 18 22:37:27.664055 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-2234decf-5d0d-4055-a0fa-bd58d7a762f5 tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:37:27.665014 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-2234decf-5d0d-4055-a0fa-bd58d7a762f5 tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:37:27.665481 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-2234decf-5d0d-4055-a0fa-bd58d7a762f5 tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:37:27.840921 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-2234decf-5d0d-4055-a0fa-bd58d7a762f5 tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:37:27.841356 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-2234decf-5d0d-4055-a0fa-bd58d7a762f5 tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:37:27.841747 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-2234decf-5d0d-4055-a0fa-bd58d7a762f5 tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:37:27.842407 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-2234decf-5d0d-4055-a0fa-bd58d7a762f5 tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:37:27.842825 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-2234decf-5d0d-4055-a0fa-bd58d7a762f5 tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:37:27.843217 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-2234decf-5d0d-4055-a0fa-bd58d7a762f5 tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:37:27.860844 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-2234decf-5d0d-4055-a0fa-bd58d7a762f5 tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Getting quotas for project 59dde594f2d04d40886a969d353b0bc5. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 18 22:37:27.864884 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-2234decf-5d0d-4055-a0fa-bd58d7a762f5 tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Getting quotas for user 372a102cdb374e8e8d8f0ebe31c78e58 and project 59dde594f2d04d40886a969d353b0bc5. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 18 22:37:27.876001 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-2234decf-5d0d-4055-a0fa-bd58d7a762f5 tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] [instance: 2dcf0d65-2127-45ea-9b81-25e7fdf74f7f] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120658) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 18 22:37:27.876807 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-2234decf-5d0d-4055-a0fa-bd58d7a762f5 tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:37:27.877243 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-2234decf-5d0d-4055-a0fa-bd58d7a762f5 tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:37:27.877632 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-2234decf-5d0d-4055-a0fa-bd58d7a762f5 tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:37:27.985509 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-2234decf-5d0d-4055-a0fa-bd58d7a762f5 tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] [instance: 2dcf0d65-2127-45ea-9b81-25e7fdf74f7f] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='87b0fec9-a948-41f9-a3c0-f0a6a5a8f1e7',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120658) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 18 22:37:27.987574 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-2234decf-5d0d-4055-a0fa-bd58d7a762f5 tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:37:27.988434 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-2234decf-5d0d-4055-a0fa-bd58d7a762f5 tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:37:27.989109 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-2234decf-5d0d-4055-a0fa-bd58d7a762f5 tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:37:28.281776 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-2234decf-5d0d-4055-a0fa-bd58d7a762f5 tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:37:28.282220 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-2234decf-5d0d-4055-a0fa-bd58d7a762f5 tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:37:28.282609 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-2234decf-5d0d-4055-a0fa-bd58d7a762f5 tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:37:28.283115 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-2234decf-5d0d-4055-a0fa-bd58d7a762f5 tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Acquiring lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 18 22:37:28.283454 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-2234decf-5d0d-4055-a0fa-bd58d7a762f5 tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Acquired lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 18 22:37:28.284352 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-64bc3a4d-4633-40b1-9b36-9ce8d7c8029c None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:37:28.284753 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-64bc3a4d-4633-40b1-9b36-9ce8d7c8029c None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:37:28.285318 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-64bc3a4d-4633-40b1-9b36-9ce8d7c8029c None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:37:28.286109 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-64bc3a4d-4633-40b1-9b36-9ce8d7c8029c None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:37:28.286513 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-64bc3a4d-4633-40b1-9b36-9ce8d7c8029c None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:37:28.286884 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-64bc3a4d-4633-40b1-9b36-9ce8d7c8029c None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:37:28.296610 n-d-853704-1 nova-conductor[120658]: ERROR nova.compute.rpcapi [None req-64bc3a4d-4633-40b1-9b36-9ce8d7c8029c None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 18 22:37:28.296993 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-64bc3a4d-4633-40b1-9b36-9ce8d7c8029c None None] Releasing lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 18 22:37:28.298217 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server [None req-64bc3a4d-4633-40b1-9b36-9ce8d7c8029c None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:37:28.298217 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:37:28.298217 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 18 22:37:28.298217 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 18 22:37:28.298217 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 18 22:37:28.298217 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 22:37:28.298217 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 18 22:37:28.298217 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 22:37:28.298217 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:37:28.298217 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 18 22:37:28.298217 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 18 22:37:28.298217 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 18 22:37:28.298217 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 18 22:37:28.298217 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 18 22:37:28.298217 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 18 22:37:28.298217 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 18 22:37:28.298217 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 18 22:37:28.298217 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 18 22:37:28.298217 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 18 22:37:28.298217 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 18 22:37:28.298217 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 18 22:37:28.298217 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 18 22:37:28.298217 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 18 22:37:28.298217 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:37:28.298217 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 22:37:33.152625 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=6,nova_cell0:UPDATE=3 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:37:38.090316 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=13,nova_api:UPDATE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:37:38.194365 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=11,nova_api:DELETE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:37:38.294876 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:37:38.295502 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell1:SELECT=11,nova_cell1:SAVEPOINT=1,nova_cell1:INSERT=18,nova_cell1:RELEASE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:37:52.963820 n-d-853704-1 nova-conductor[120657]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 18 22:37:53.166053 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=15,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:37:57.599896 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-a46a4dc7-3f6f-4265-afef-65c2ad4139c5 tempest-VolumesBackupsTest-1065392879 tempest-VolumesBackupsTest-1065392879-project] Took 0.60 seconds to select destinations for 1 instance(s). {{(pid=120657) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 18 22:37:57.620161 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-a46a4dc7-3f6f-4265-afef-65c2ad4139c5 tempest-VolumesBackupsTest-1065392879 tempest-VolumesBackupsTest-1065392879-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:37:57.620594 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-a46a4dc7-3f6f-4265-afef-65c2ad4139c5 tempest-VolumesBackupsTest-1065392879 tempest-VolumesBackupsTest-1065392879-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:37:57.620988 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-a46a4dc7-3f6f-4265-afef-65c2ad4139c5 tempest-VolumesBackupsTest-1065392879 tempest-VolumesBackupsTest-1065392879-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:37:57.816480 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-a46a4dc7-3f6f-4265-afef-65c2ad4139c5 tempest-VolumesBackupsTest-1065392879 tempest-VolumesBackupsTest-1065392879-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:37:57.816926 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-a46a4dc7-3f6f-4265-afef-65c2ad4139c5 tempest-VolumesBackupsTest-1065392879 tempest-VolumesBackupsTest-1065392879-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:37:57.817316 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-a46a4dc7-3f6f-4265-afef-65c2ad4139c5 tempest-VolumesBackupsTest-1065392879 tempest-VolumesBackupsTest-1065392879-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:37:57.817977 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-a46a4dc7-3f6f-4265-afef-65c2ad4139c5 tempest-VolumesBackupsTest-1065392879 tempest-VolumesBackupsTest-1065392879-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:37:57.818390 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-a46a4dc7-3f6f-4265-afef-65c2ad4139c5 tempest-VolumesBackupsTest-1065392879 tempest-VolumesBackupsTest-1065392879-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:37:57.818776 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-a46a4dc7-3f6f-4265-afef-65c2ad4139c5 tempest-VolumesBackupsTest-1065392879 tempest-VolumesBackupsTest-1065392879-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:37:57.834220 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-a46a4dc7-3f6f-4265-afef-65c2ad4139c5 tempest-VolumesBackupsTest-1065392879 tempest-VolumesBackupsTest-1065392879-project] Getting quotas for project 170e46c4b06b4f248ff107814f9ff138. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 18 22:37:57.838191 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-a46a4dc7-3f6f-4265-afef-65c2ad4139c5 tempest-VolumesBackupsTest-1065392879 tempest-VolumesBackupsTest-1065392879-project] Getting quotas for user 4df1ed3f8e4547cb8d55ff2a4c41bc0e and project 170e46c4b06b4f248ff107814f9ff138. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 18 22:37:57.848368 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-a46a4dc7-3f6f-4265-afef-65c2ad4139c5 tempest-VolumesBackupsTest-1065392879 tempest-VolumesBackupsTest-1065392879-project] [instance: de5a961d-4e40-4c2a-8a4b-62e0ba4e9e63] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120657) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 18 22:37:57.849117 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-a46a4dc7-3f6f-4265-afef-65c2ad4139c5 tempest-VolumesBackupsTest-1065392879 tempest-VolumesBackupsTest-1065392879-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:37:57.849544 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-a46a4dc7-3f6f-4265-afef-65c2ad4139c5 tempest-VolumesBackupsTest-1065392879 tempest-VolumesBackupsTest-1065392879-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:37:57.849935 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-a46a4dc7-3f6f-4265-afef-65c2ad4139c5 tempest-VolumesBackupsTest-1065392879 tempest-VolumesBackupsTest-1065392879-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:37:57.925743 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-a46a4dc7-3f6f-4265-afef-65c2ad4139c5 tempest-VolumesBackupsTest-1065392879 tempest-VolumesBackupsTest-1065392879-project] [instance: de5a961d-4e40-4c2a-8a4b-62e0ba4e9e63] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='87b0fec9-a948-41f9-a3c0-f0a6a5a8f1e7',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120657) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 18 22:37:57.926960 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-a46a4dc7-3f6f-4265-afef-65c2ad4139c5 tempest-VolumesBackupsTest-1065392879 tempest-VolumesBackupsTest-1065392879-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:37:57.927395 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-a46a4dc7-3f6f-4265-afef-65c2ad4139c5 tempest-VolumesBackupsTest-1065392879 tempest-VolumesBackupsTest-1065392879-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:37:57.927812 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-a46a4dc7-3f6f-4265-afef-65c2ad4139c5 tempest-VolumesBackupsTest-1065392879 tempest-VolumesBackupsTest-1065392879-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:37:58.160909 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-a46a4dc7-3f6f-4265-afef-65c2ad4139c5 tempest-VolumesBackupsTest-1065392879 tempest-VolumesBackupsTest-1065392879-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:37:58.161378 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-a46a4dc7-3f6f-4265-afef-65c2ad4139c5 tempest-VolumesBackupsTest-1065392879 tempest-VolumesBackupsTest-1065392879-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:37:58.161794 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-a46a4dc7-3f6f-4265-afef-65c2ad4139c5 tempest-VolumesBackupsTest-1065392879 tempest-VolumesBackupsTest-1065392879-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:37:58.162247 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-a46a4dc7-3f6f-4265-afef-65c2ad4139c5 tempest-VolumesBackupsTest-1065392879 tempest-VolumesBackupsTest-1065392879-project] Acquiring lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 18 22:37:58.162601 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-a46a4dc7-3f6f-4265-afef-65c2ad4139c5 tempest-VolumesBackupsTest-1065392879 tempest-VolumesBackupsTest-1065392879-project] Acquired lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 18 22:37:58.163341 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-52ce5f8c-027b-47b2-9eb4-cafa84fca6a8 None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:37:58.163790 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-52ce5f8c-027b-47b2-9eb4-cafa84fca6a8 None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:37:58.164205 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-52ce5f8c-027b-47b2-9eb4-cafa84fca6a8 None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:37:58.164852 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-52ce5f8c-027b-47b2-9eb4-cafa84fca6a8 None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:37:58.165257 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-52ce5f8c-027b-47b2-9eb4-cafa84fca6a8 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:37:58.165638 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-52ce5f8c-027b-47b2-9eb4-cafa84fca6a8 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:37:58.174471 n-d-853704-1 nova-conductor[120657]: ERROR nova.compute.rpcapi [None req-52ce5f8c-027b-47b2-9eb4-cafa84fca6a8 None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 18 22:37:58.174854 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-52ce5f8c-027b-47b2-9eb4-cafa84fca6a8 None None] Releasing lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 18 22:37:58.175784 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server [None req-52ce5f8c-027b-47b2-9eb4-cafa84fca6a8 None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:37:58.175784 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:37:58.175784 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 18 22:37:58.175784 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 18 22:37:58.175784 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 18 22:37:58.175784 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 22:37:58.175784 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 18 22:37:58.175784 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 22:37:58.175784 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:37:58.175784 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 18 22:37:58.175784 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 18 22:37:58.175784 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 18 22:37:58.175784 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 18 22:37:58.175784 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 18 22:37:58.175784 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 18 22:37:58.175784 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 18 22:37:58.175784 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 18 22:37:58.175784 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 18 22:37:58.175784 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 18 22:37:58.175784 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 18 22:37:58.175784 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 18 22:37:58.175784 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 18 22:37:58.175784 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 18 22:37:58.175784 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:37:58.175784 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 22:38:08.076292 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=12 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:38:08.077820 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=12,nova_api:UPDATE=1,nova_api:DELETE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:38:08.171367 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:38:08.172154 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell1:SELECT=11,nova_cell1:SAVEPOINT=1,nova_cell1:INSERT=18,nova_cell1:RELEASE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:38:12.118496 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-a2c4fb9f-2b07-4561-b311-c208c7d2d4be tempest-ServerShowV247Test-1014934114 tempest-ServerShowV247Test-1014934114-project] Took 0.53 seconds to select destinations for 1 instance(s). {{(pid=120658) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 18 22:38:12.142330 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-a2c4fb9f-2b07-4561-b311-c208c7d2d4be tempest-ServerShowV247Test-1014934114 tempest-ServerShowV247Test-1014934114-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:38:12.142877 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-a2c4fb9f-2b07-4561-b311-c208c7d2d4be tempest-ServerShowV247Test-1014934114 tempest-ServerShowV247Test-1014934114-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:38:12.143325 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-a2c4fb9f-2b07-4561-b311-c208c7d2d4be tempest-ServerShowV247Test-1014934114 tempest-ServerShowV247Test-1014934114-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:38:12.352564 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-a2c4fb9f-2b07-4561-b311-c208c7d2d4be tempest-ServerShowV247Test-1014934114 tempest-ServerShowV247Test-1014934114-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:38:12.353070 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-a2c4fb9f-2b07-4561-b311-c208c7d2d4be tempest-ServerShowV247Test-1014934114 tempest-ServerShowV247Test-1014934114-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:38:12.353527 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-a2c4fb9f-2b07-4561-b311-c208c7d2d4be tempest-ServerShowV247Test-1014934114 tempest-ServerShowV247Test-1014934114-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:38:12.354297 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-a2c4fb9f-2b07-4561-b311-c208c7d2d4be tempest-ServerShowV247Test-1014934114 tempest-ServerShowV247Test-1014934114-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:38:12.354770 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-a2c4fb9f-2b07-4561-b311-c208c7d2d4be tempest-ServerShowV247Test-1014934114 tempest-ServerShowV247Test-1014934114-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:38:12.355337 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-a2c4fb9f-2b07-4561-b311-c208c7d2d4be tempest-ServerShowV247Test-1014934114 tempest-ServerShowV247Test-1014934114-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:38:12.371608 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-a2c4fb9f-2b07-4561-b311-c208c7d2d4be tempest-ServerShowV247Test-1014934114 tempest-ServerShowV247Test-1014934114-project] Getting quotas for project 26022e5c0a044fbea639c45a8a2c8af8. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 18 22:38:12.376499 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-a2c4fb9f-2b07-4561-b311-c208c7d2d4be tempest-ServerShowV247Test-1014934114 tempest-ServerShowV247Test-1014934114-project] Getting quotas for user 9f1b45f54bf8407a9bf92a8b420f5ee0 and project 26022e5c0a044fbea639c45a8a2c8af8. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 18 22:38:12.387208 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-a2c4fb9f-2b07-4561-b311-c208c7d2d4be tempest-ServerShowV247Test-1014934114 tempest-ServerShowV247Test-1014934114-project] [instance: d957ceb9-8ac8-4575-b7cf-bc8d72350439] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120658) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 18 22:38:12.388765 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-a2c4fb9f-2b07-4561-b311-c208c7d2d4be tempest-ServerShowV247Test-1014934114 tempest-ServerShowV247Test-1014934114-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:38:12.389414 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-a2c4fb9f-2b07-4561-b311-c208c7d2d4be tempest-ServerShowV247Test-1014934114 tempest-ServerShowV247Test-1014934114-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:38:12.389774 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-a2c4fb9f-2b07-4561-b311-c208c7d2d4be tempest-ServerShowV247Test-1014934114 tempest-ServerShowV247Test-1014934114-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:38:12.462018 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-a2c4fb9f-2b07-4561-b311-c208c7d2d4be tempest-ServerShowV247Test-1014934114 tempest-ServerShowV247Test-1014934114-project] [instance: d957ceb9-8ac8-4575-b7cf-bc8d72350439] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='87b0fec9-a948-41f9-a3c0-f0a6a5a8f1e7',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120658) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 18 22:38:12.463229 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-a2c4fb9f-2b07-4561-b311-c208c7d2d4be tempest-ServerShowV247Test-1014934114 tempest-ServerShowV247Test-1014934114-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:38:12.463678 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-a2c4fb9f-2b07-4561-b311-c208c7d2d4be tempest-ServerShowV247Test-1014934114 tempest-ServerShowV247Test-1014934114-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:38:12.464078 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-a2c4fb9f-2b07-4561-b311-c208c7d2d4be tempest-ServerShowV247Test-1014934114 tempest-ServerShowV247Test-1014934114-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:38:12.525918 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-75b1d1e8-4719-4542-973d-c09184de38f7 tempest-ServerShowV247Test-1014934114 tempest-ServerShowV247Test-1014934114-project] Took 0.43 seconds to select destinations for 1 instance(s). {{(pid=120657) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 18 22:38:12.544856 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-75b1d1e8-4719-4542-973d-c09184de38f7 tempest-ServerShowV247Test-1014934114 tempest-ServerShowV247Test-1014934114-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:38:12.545294 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-75b1d1e8-4719-4542-973d-c09184de38f7 tempest-ServerShowV247Test-1014934114 tempest-ServerShowV247Test-1014934114-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:38:12.545683 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-75b1d1e8-4719-4542-973d-c09184de38f7 tempest-ServerShowV247Test-1014934114 tempest-ServerShowV247Test-1014934114-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:38:12.675410 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=15,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:38:12.678798 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-75b1d1e8-4719-4542-973d-c09184de38f7 tempest-ServerShowV247Test-1014934114 tempest-ServerShowV247Test-1014934114-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:38:12.679234 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-75b1d1e8-4719-4542-973d-c09184de38f7 tempest-ServerShowV247Test-1014934114 tempest-ServerShowV247Test-1014934114-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:38:12.679621 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-75b1d1e8-4719-4542-973d-c09184de38f7 tempest-ServerShowV247Test-1014934114 tempest-ServerShowV247Test-1014934114-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:38:12.680300 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-75b1d1e8-4719-4542-973d-c09184de38f7 tempest-ServerShowV247Test-1014934114 tempest-ServerShowV247Test-1014934114-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:38:12.680703 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-75b1d1e8-4719-4542-973d-c09184de38f7 tempest-ServerShowV247Test-1014934114 tempest-ServerShowV247Test-1014934114-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:38:12.681079 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-75b1d1e8-4719-4542-973d-c09184de38f7 tempest-ServerShowV247Test-1014934114 tempest-ServerShowV247Test-1014934114-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:38:12.696702 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-75b1d1e8-4719-4542-973d-c09184de38f7 tempest-ServerShowV247Test-1014934114 tempest-ServerShowV247Test-1014934114-project] Getting quotas for project 26022e5c0a044fbea639c45a8a2c8af8. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 18 22:38:12.700968 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-75b1d1e8-4719-4542-973d-c09184de38f7 tempest-ServerShowV247Test-1014934114 tempest-ServerShowV247Test-1014934114-project] Getting quotas for user 9f1b45f54bf8407a9bf92a8b420f5ee0 and project 26022e5c0a044fbea639c45a8a2c8af8. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 18 22:38:12.711505 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-75b1d1e8-4719-4542-973d-c09184de38f7 tempest-ServerShowV247Test-1014934114 tempest-ServerShowV247Test-1014934114-project] [instance: 3f227936-d569-41b3-8743-b9fe51059a3e] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120657) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 18 22:38:12.712306 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-75b1d1e8-4719-4542-973d-c09184de38f7 tempest-ServerShowV247Test-1014934114 tempest-ServerShowV247Test-1014934114-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:38:12.713198 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-75b1d1e8-4719-4542-973d-c09184de38f7 tempest-ServerShowV247Test-1014934114 tempest-ServerShowV247Test-1014934114-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:38:12.713581 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-75b1d1e8-4719-4542-973d-c09184de38f7 tempest-ServerShowV247Test-1014934114 tempest-ServerShowV247Test-1014934114-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:38:12.720163 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-a2c4fb9f-2b07-4561-b311-c208c7d2d4be tempest-ServerShowV247Test-1014934114 tempest-ServerShowV247Test-1014934114-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:38:12.720618 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-a2c4fb9f-2b07-4561-b311-c208c7d2d4be tempest-ServerShowV247Test-1014934114 tempest-ServerShowV247Test-1014934114-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:38:12.721025 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-a2c4fb9f-2b07-4561-b311-c208c7d2d4be tempest-ServerShowV247Test-1014934114 tempest-ServerShowV247Test-1014934114-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:38:12.725012 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-a2c4fb9f-2b07-4561-b311-c208c7d2d4be tempest-ServerShowV247Test-1014934114 tempest-ServerShowV247Test-1014934114-project] Acquiring lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 18 22:38:12.725012 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-a2c4fb9f-2b07-4561-b311-c208c7d2d4be tempest-ServerShowV247Test-1014934114 tempest-ServerShowV247Test-1014934114-project] Acquired lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 18 22:38:12.725012 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-54efa760-5214-4a10-a29d-c0d8ded5f6ed None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:38:12.725012 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-54efa760-5214-4a10-a29d-c0d8ded5f6ed None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:38:12.725012 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-54efa760-5214-4a10-a29d-c0d8ded5f6ed None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:38:12.725012 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-54efa760-5214-4a10-a29d-c0d8ded5f6ed None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:38:12.725012 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-54efa760-5214-4a10-a29d-c0d8ded5f6ed None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:38:12.725012 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-54efa760-5214-4a10-a29d-c0d8ded5f6ed None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:38:12.734142 n-d-853704-1 nova-conductor[120658]: ERROR nova.compute.rpcapi [None req-54efa760-5214-4a10-a29d-c0d8ded5f6ed None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 18 22:38:12.734513 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-54efa760-5214-4a10-a29d-c0d8ded5f6ed None None] Releasing lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 18 22:38:12.735409 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server [None req-54efa760-5214-4a10-a29d-c0d8ded5f6ed None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:38:12.735409 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:38:12.735409 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 18 22:38:12.735409 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 18 22:38:12.735409 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 18 22:38:12.735409 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 22:38:12.735409 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 18 22:38:12.735409 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 22:38:12.735409 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:38:12.735409 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 18 22:38:12.735409 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 18 22:38:12.735409 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 18 22:38:12.735409 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 18 22:38:12.735409 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 18 22:38:12.735409 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 18 22:38:12.735409 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 18 22:38:12.735409 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 18 22:38:12.735409 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 18 22:38:12.735409 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 18 22:38:12.735409 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 18 22:38:12.735409 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 18 22:38:12.735409 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 18 22:38:12.735409 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 18 22:38:12.735409 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:38:12.735409 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 22:38:12.788193 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-75b1d1e8-4719-4542-973d-c09184de38f7 tempest-ServerShowV247Test-1014934114 tempest-ServerShowV247Test-1014934114-project] [instance: 3f227936-d569-41b3-8743-b9fe51059a3e] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='87b0fec9-a948-41f9-a3c0-f0a6a5a8f1e7',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120657) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 18 22:38:12.789433 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-75b1d1e8-4719-4542-973d-c09184de38f7 tempest-ServerShowV247Test-1014934114 tempest-ServerShowV247Test-1014934114-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:38:12.789876 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-75b1d1e8-4719-4542-973d-c09184de38f7 tempest-ServerShowV247Test-1014934114 tempest-ServerShowV247Test-1014934114-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:38:12.790274 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-75b1d1e8-4719-4542-973d-c09184de38f7 tempest-ServerShowV247Test-1014934114 tempest-ServerShowV247Test-1014934114-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:38:13.005691 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-75b1d1e8-4719-4542-973d-c09184de38f7 tempest-ServerShowV247Test-1014934114 tempest-ServerShowV247Test-1014934114-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:38:13.006153 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-75b1d1e8-4719-4542-973d-c09184de38f7 tempest-ServerShowV247Test-1014934114 tempest-ServerShowV247Test-1014934114-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:38:13.006546 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-75b1d1e8-4719-4542-973d-c09184de38f7 tempest-ServerShowV247Test-1014934114 tempest-ServerShowV247Test-1014934114-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:38:13.006995 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-75b1d1e8-4719-4542-973d-c09184de38f7 tempest-ServerShowV247Test-1014934114 tempest-ServerShowV247Test-1014934114-project] Acquiring lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 18 22:38:13.007342 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-75b1d1e8-4719-4542-973d-c09184de38f7 tempest-ServerShowV247Test-1014934114 tempest-ServerShowV247Test-1014934114-project] Acquired lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 18 22:38:13.008123 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-85b6ee2d-4505-43da-ac0f-f8de23128341 None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:38:13.008531 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-85b6ee2d-4505-43da-ac0f-f8de23128341 None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:38:13.008913 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-85b6ee2d-4505-43da-ac0f-f8de23128341 None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:38:13.009550 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-85b6ee2d-4505-43da-ac0f-f8de23128341 None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:38:13.010001 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-85b6ee2d-4505-43da-ac0f-f8de23128341 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:38:13.010375 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-85b6ee2d-4505-43da-ac0f-f8de23128341 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:38:13.019687 n-d-853704-1 nova-conductor[120657]: ERROR nova.compute.rpcapi [None req-85b6ee2d-4505-43da-ac0f-f8de23128341 None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 18 22:38:13.020081 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-85b6ee2d-4505-43da-ac0f-f8de23128341 None None] Releasing lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 18 22:38:13.020971 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server [None req-85b6ee2d-4505-43da-ac0f-f8de23128341 None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:38:13.020971 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:38:13.020971 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 18 22:38:13.020971 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 18 22:38:13.020971 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 18 22:38:13.020971 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 22:38:13.020971 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 18 22:38:13.020971 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 22:38:13.020971 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:38:13.020971 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 18 22:38:13.020971 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 18 22:38:13.020971 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 18 22:38:13.020971 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 18 22:38:13.020971 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 18 22:38:13.020971 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 18 22:38:13.020971 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 18 22:38:13.020971 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 18 22:38:13.020971 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 18 22:38:13.020971 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 18 22:38:13.020971 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 18 22:38:13.020971 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 18 22:38:13.020971 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 18 22:38:13.020971 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 18 22:38:13.020971 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:38:13.020971 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 22:38:22.553687 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=11,nova_api:UPDATE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:38:22.636169 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=13,nova_api:DELETE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:38:22.731475 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:38:22.732366 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell1:SELECT=11,nova_cell1:SAVEPOINT=1,nova_cell1:INSERT=18,nova_cell1:RELEASE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:38:22.856212 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=14 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:38:22.933616 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=10,nova_api:UPDATE=1,nova_api:DELETE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:38:23.016591 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:38:23.017361 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell1:SELECT=11,nova_cell1:INSERT=17 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:38:23.171397 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=11,nova_cell0:UPDATE=3 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:38:43.146030 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=12,nova_cell0:UPDATE=3 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:39:03.150706 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=3,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:39:03.171475 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:39:16.783308 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-de93ac46-1378-405a-8546-2902defae374 tempest-AttachSCSIVolumeTestJSON-1860771682 tempest-AttachSCSIVolumeTestJSON-1860771682-project] Took 0.53 seconds to select destinations for 1 instance(s). {{(pid=120658) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 18 22:39:16.803235 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-de93ac46-1378-405a-8546-2902defae374 tempest-AttachSCSIVolumeTestJSON-1860771682 tempest-AttachSCSIVolumeTestJSON-1860771682-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:39:16.803686 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-de93ac46-1378-405a-8546-2902defae374 tempest-AttachSCSIVolumeTestJSON-1860771682 tempest-AttachSCSIVolumeTestJSON-1860771682-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:39:16.804118 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-de93ac46-1378-405a-8546-2902defae374 tempest-AttachSCSIVolumeTestJSON-1860771682 tempest-AttachSCSIVolumeTestJSON-1860771682-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:39:17.097582 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-de93ac46-1378-405a-8546-2902defae374 tempest-AttachSCSIVolumeTestJSON-1860771682 tempest-AttachSCSIVolumeTestJSON-1860771682-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:39:17.098156 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-de93ac46-1378-405a-8546-2902defae374 tempest-AttachSCSIVolumeTestJSON-1860771682 tempest-AttachSCSIVolumeTestJSON-1860771682-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:39:17.098540 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-de93ac46-1378-405a-8546-2902defae374 tempest-AttachSCSIVolumeTestJSON-1860771682 tempest-AttachSCSIVolumeTestJSON-1860771682-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:39:17.099324 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-de93ac46-1378-405a-8546-2902defae374 tempest-AttachSCSIVolumeTestJSON-1860771682 tempest-AttachSCSIVolumeTestJSON-1860771682-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:39:17.099682 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-de93ac46-1378-405a-8546-2902defae374 tempest-AttachSCSIVolumeTestJSON-1860771682 tempest-AttachSCSIVolumeTestJSON-1860771682-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:39:17.100062 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-de93ac46-1378-405a-8546-2902defae374 tempest-AttachSCSIVolumeTestJSON-1860771682 tempest-AttachSCSIVolumeTestJSON-1860771682-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:39:17.117432 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-de93ac46-1378-405a-8546-2902defae374 tempest-AttachSCSIVolumeTestJSON-1860771682 tempest-AttachSCSIVolumeTestJSON-1860771682-project] Getting quotas for project 2927889181624505bb822ca3b8dffb1f. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 18 22:39:17.122764 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-de93ac46-1378-405a-8546-2902defae374 tempest-AttachSCSIVolumeTestJSON-1860771682 tempest-AttachSCSIVolumeTestJSON-1860771682-project] Getting quotas for user d6b03c716b6f48038ef419b1774f4e6d and project 2927889181624505bb822ca3b8dffb1f. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 18 22:39:17.134625 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-de93ac46-1378-405a-8546-2902defae374 tempest-AttachSCSIVolumeTestJSON-1860771682 tempest-AttachSCSIVolumeTestJSON-1860771682-project] [instance: d2e0713f-4da4-4382-916d-ad472eb38676] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120658) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 18 22:39:17.135389 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-de93ac46-1378-405a-8546-2902defae374 tempest-AttachSCSIVolumeTestJSON-1860771682 tempest-AttachSCSIVolumeTestJSON-1860771682-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:39:17.135860 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-de93ac46-1378-405a-8546-2902defae374 tempest-AttachSCSIVolumeTestJSON-1860771682 tempest-AttachSCSIVolumeTestJSON-1860771682-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:39:17.136248 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-de93ac46-1378-405a-8546-2902defae374 tempest-AttachSCSIVolumeTestJSON-1860771682 tempest-AttachSCSIVolumeTestJSON-1860771682-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:39:17.213685 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-de93ac46-1378-405a-8546-2902defae374 tempest-AttachSCSIVolumeTestJSON-1860771682 tempest-AttachSCSIVolumeTestJSON-1860771682-project] [instance: d2e0713f-4da4-4382-916d-ad472eb38676] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='be4dadc1-0ef8-47ce-b670-13dda9ce1ce5',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120658) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 18 22:39:17.214990 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-de93ac46-1378-405a-8546-2902defae374 tempest-AttachSCSIVolumeTestJSON-1860771682 tempest-AttachSCSIVolumeTestJSON-1860771682-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:39:17.215539 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-de93ac46-1378-405a-8546-2902defae374 tempest-AttachSCSIVolumeTestJSON-1860771682 tempest-AttachSCSIVolumeTestJSON-1860771682-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:39:17.215985 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-de93ac46-1378-405a-8546-2902defae374 tempest-AttachSCSIVolumeTestJSON-1860771682 tempest-AttachSCSIVolumeTestJSON-1860771682-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:39:17.380601 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-de93ac46-1378-405a-8546-2902defae374 tempest-AttachSCSIVolumeTestJSON-1860771682 tempest-AttachSCSIVolumeTestJSON-1860771682-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:39:17.381043 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-de93ac46-1378-405a-8546-2902defae374 tempest-AttachSCSIVolumeTestJSON-1860771682 tempest-AttachSCSIVolumeTestJSON-1860771682-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:39:17.381432 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-de93ac46-1378-405a-8546-2902defae374 tempest-AttachSCSIVolumeTestJSON-1860771682 tempest-AttachSCSIVolumeTestJSON-1860771682-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:39:17.381877 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-de93ac46-1378-405a-8546-2902defae374 tempest-AttachSCSIVolumeTestJSON-1860771682 tempest-AttachSCSIVolumeTestJSON-1860771682-project] Acquiring lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 18 22:39:17.382245 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-de93ac46-1378-405a-8546-2902defae374 tempest-AttachSCSIVolumeTestJSON-1860771682 tempest-AttachSCSIVolumeTestJSON-1860771682-project] Acquired lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 18 22:39:17.383011 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-d9f176fe-f16c-4f4b-8987-1f9873bea1a4 None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:39:17.383414 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-d9f176fe-f16c-4f4b-8987-1f9873bea1a4 None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:39:17.383826 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-d9f176fe-f16c-4f4b-8987-1f9873bea1a4 None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:39:17.384462 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-d9f176fe-f16c-4f4b-8987-1f9873bea1a4 None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:39:17.384861 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-d9f176fe-f16c-4f4b-8987-1f9873bea1a4 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:39:17.385235 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-d9f176fe-f16c-4f4b-8987-1f9873bea1a4 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:39:17.393849 n-d-853704-1 nova-conductor[120658]: ERROR nova.compute.rpcapi [None req-d9f176fe-f16c-4f4b-8987-1f9873bea1a4 None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 18 22:39:17.394231 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-d9f176fe-f16c-4f4b-8987-1f9873bea1a4 None None] Releasing lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 18 22:39:17.395188 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server [None req-d9f176fe-f16c-4f4b-8987-1f9873bea1a4 None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:39:17.395188 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:39:17.395188 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 18 22:39:17.395188 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 18 22:39:17.395188 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 18 22:39:17.395188 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 22:39:17.395188 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 18 22:39:17.395188 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 22:39:17.395188 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:39:17.395188 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 18 22:39:17.395188 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 18 22:39:17.395188 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 18 22:39:17.395188 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 18 22:39:17.395188 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 18 22:39:17.395188 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 18 22:39:17.395188 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 18 22:39:17.395188 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 18 22:39:17.395188 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 18 22:39:17.395188 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 18 22:39:17.395188 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 18 22:39:17.395188 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 18 22:39:17.395188 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 18 22:39:17.395188 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 18 22:39:17.395188 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:39:17.395188 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 22:39:27.329776 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=10 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:39:27.330602 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=14,nova_api:UPDATE=1,nova_api:DELETE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:39:27.391654 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:39:27.392441 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell1:SELECT=14,nova_cell1:SAVEPOINT=1,nova_cell1:INSERT=18,nova_cell1:RELEASE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:39:43.148806 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:39:43.169815 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=14,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:40:03.173992 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=3,nova_cell0:UPDATE=2 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:40:23.154010 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:40:43.172925 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:41:03.159004 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=7,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:41:12.972511 n-d-853704-1 nova-conductor[120657]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 18 22:41:16.269662 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-b40213f7-1323-4d34-a9b3-0da62a47a96d tempest-ServerRescueNegativeTestJSON-2101034428 tempest-ServerRescueNegativeTestJSON-2101034428-project] Took 0.63 seconds to select destinations for 1 instance(s). {{(pid=120657) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 18 22:41:16.298046 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-b40213f7-1323-4d34-a9b3-0da62a47a96d tempest-ServerRescueNegativeTestJSON-2101034428 tempest-ServerRescueNegativeTestJSON-2101034428-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:41:16.299642 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-b40213f7-1323-4d34-a9b3-0da62a47a96d tempest-ServerRescueNegativeTestJSON-2101034428 tempest-ServerRescueNegativeTestJSON-2101034428-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:41:16.299642 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-b40213f7-1323-4d34-a9b3-0da62a47a96d tempest-ServerRescueNegativeTestJSON-2101034428 tempest-ServerRescueNegativeTestJSON-2101034428-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:41:16.577927 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-b40213f7-1323-4d34-a9b3-0da62a47a96d tempest-ServerRescueNegativeTestJSON-2101034428 tempest-ServerRescueNegativeTestJSON-2101034428-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:41:16.578386 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-b40213f7-1323-4d34-a9b3-0da62a47a96d tempest-ServerRescueNegativeTestJSON-2101034428 tempest-ServerRescueNegativeTestJSON-2101034428-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:41:16.578888 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-b40213f7-1323-4d34-a9b3-0da62a47a96d tempest-ServerRescueNegativeTestJSON-2101034428 tempest-ServerRescueNegativeTestJSON-2101034428-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:41:16.579695 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-b40213f7-1323-4d34-a9b3-0da62a47a96d tempest-ServerRescueNegativeTestJSON-2101034428 tempest-ServerRescueNegativeTestJSON-2101034428-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:41:16.580119 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-b40213f7-1323-4d34-a9b3-0da62a47a96d tempest-ServerRescueNegativeTestJSON-2101034428 tempest-ServerRescueNegativeTestJSON-2101034428-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:41:16.580511 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-b40213f7-1323-4d34-a9b3-0da62a47a96d tempest-ServerRescueNegativeTestJSON-2101034428 tempest-ServerRescueNegativeTestJSON-2101034428-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:41:16.614330 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-b40213f7-1323-4d34-a9b3-0da62a47a96d tempest-ServerRescueNegativeTestJSON-2101034428 tempest-ServerRescueNegativeTestJSON-2101034428-project] Getting quotas for project c3a16922c1744d71bf7f99aa006f35e7. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 18 22:41:16.624019 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-b40213f7-1323-4d34-a9b3-0da62a47a96d tempest-ServerRescueNegativeTestJSON-2101034428 tempest-ServerRescueNegativeTestJSON-2101034428-project] Getting quotas for user 3c90eef5582b416d929dbafbd4442f34 and project c3a16922c1744d71bf7f99aa006f35e7. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 18 22:41:16.651189 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-b40213f7-1323-4d34-a9b3-0da62a47a96d tempest-ServerRescueNegativeTestJSON-2101034428 tempest-ServerRescueNegativeTestJSON-2101034428-project] [instance: ec918a47-085d-4722-b929-565a52f92e94] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120657) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 18 22:41:16.652015 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-b40213f7-1323-4d34-a9b3-0da62a47a96d tempest-ServerRescueNegativeTestJSON-2101034428 tempest-ServerRescueNegativeTestJSON-2101034428-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:41:16.652460 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-b40213f7-1323-4d34-a9b3-0da62a47a96d tempest-ServerRescueNegativeTestJSON-2101034428 tempest-ServerRescueNegativeTestJSON-2101034428-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:41:16.652854 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-b40213f7-1323-4d34-a9b3-0da62a47a96d tempest-ServerRescueNegativeTestJSON-2101034428 tempest-ServerRescueNegativeTestJSON-2101034428-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:41:16.746740 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-b40213f7-1323-4d34-a9b3-0da62a47a96d tempest-ServerRescueNegativeTestJSON-2101034428 tempest-ServerRescueNegativeTestJSON-2101034428-project] [instance: ec918a47-085d-4722-b929-565a52f92e94] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='87b0fec9-a948-41f9-a3c0-f0a6a5a8f1e7',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120657) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 18 22:41:16.751377 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-b40213f7-1323-4d34-a9b3-0da62a47a96d tempest-ServerRescueNegativeTestJSON-2101034428 tempest-ServerRescueNegativeTestJSON-2101034428-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:41:16.751377 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-b40213f7-1323-4d34-a9b3-0da62a47a96d tempest-ServerRescueNegativeTestJSON-2101034428 tempest-ServerRescueNegativeTestJSON-2101034428-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:41:16.751377 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-b40213f7-1323-4d34-a9b3-0da62a47a96d tempest-ServerRescueNegativeTestJSON-2101034428 tempest-ServerRescueNegativeTestJSON-2101034428-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:41:17.004638 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-b40213f7-1323-4d34-a9b3-0da62a47a96d tempest-ServerRescueNegativeTestJSON-2101034428 tempest-ServerRescueNegativeTestJSON-2101034428-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:41:17.005132 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-b40213f7-1323-4d34-a9b3-0da62a47a96d tempest-ServerRescueNegativeTestJSON-2101034428 tempest-ServerRescueNegativeTestJSON-2101034428-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:41:17.005521 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-b40213f7-1323-4d34-a9b3-0da62a47a96d tempest-ServerRescueNegativeTestJSON-2101034428 tempest-ServerRescueNegativeTestJSON-2101034428-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:41:17.005994 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-b40213f7-1323-4d34-a9b3-0da62a47a96d tempest-ServerRescueNegativeTestJSON-2101034428 tempest-ServerRescueNegativeTestJSON-2101034428-project] Acquiring lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 18 22:41:17.006422 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-b40213f7-1323-4d34-a9b3-0da62a47a96d tempest-ServerRescueNegativeTestJSON-2101034428 tempest-ServerRescueNegativeTestJSON-2101034428-project] Acquired lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 18 22:41:17.007263 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-6a503d45-e826-47cd-b4d1-eddd549abc29 None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:41:17.007703 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-6a503d45-e826-47cd-b4d1-eddd549abc29 None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:41:17.010621 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-6a503d45-e826-47cd-b4d1-eddd549abc29 None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.003s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:41:17.011483 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-6a503d45-e826-47cd-b4d1-eddd549abc29 None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:41:17.012262 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-6a503d45-e826-47cd-b4d1-eddd549abc29 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:41:17.018842 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-6a503d45-e826-47cd-b4d1-eddd549abc29 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:41:17.033378 n-d-853704-1 nova-conductor[120657]: ERROR nova.compute.rpcapi [None req-6a503d45-e826-47cd-b4d1-eddd549abc29 None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 18 22:41:17.033797 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-6a503d45-e826-47cd-b4d1-eddd549abc29 None None] Releasing lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 18 22:41:17.036238 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server [None req-6a503d45-e826-47cd-b4d1-eddd549abc29 None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:41:17.036238 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:41:17.036238 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 18 22:41:17.036238 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 18 22:41:17.036238 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 18 22:41:17.036238 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 22:41:17.036238 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 18 22:41:17.036238 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 22:41:17.036238 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:41:17.036238 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 18 22:41:17.036238 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 18 22:41:17.036238 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 18 22:41:17.036238 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 18 22:41:17.036238 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 18 22:41:17.036238 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 18 22:41:17.036238 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 18 22:41:17.036238 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 18 22:41:17.036238 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 18 22:41:17.036238 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 18 22:41:17.036238 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 18 22:41:17.036238 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 18 22:41:17.036238 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 18 22:41:17.036238 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 18 22:41:17.036238 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:41:17.036238 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 22:41:17.505556 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-fe358061-27b8-4d97-b7f9-c2e77a64808c tempest-ServerRescueNegativeTestJSON-2101034428 tempest-ServerRescueNegativeTestJSON-2101034428-project] Took 0.53 seconds to select destinations for 1 instance(s). {{(pid=120658) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 18 22:41:17.526934 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-fe358061-27b8-4d97-b7f9-c2e77a64808c tempest-ServerRescueNegativeTestJSON-2101034428 tempest-ServerRescueNegativeTestJSON-2101034428-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:41:17.527885 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-fe358061-27b8-4d97-b7f9-c2e77a64808c tempest-ServerRescueNegativeTestJSON-2101034428 tempest-ServerRescueNegativeTestJSON-2101034428-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:41:17.528953 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-fe358061-27b8-4d97-b7f9-c2e77a64808c tempest-ServerRescueNegativeTestJSON-2101034428 tempest-ServerRescueNegativeTestJSON-2101034428-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:41:17.631305 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:41:17.633617 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-fe358061-27b8-4d97-b7f9-c2e77a64808c tempest-ServerRescueNegativeTestJSON-2101034428 tempest-ServerRescueNegativeTestJSON-2101034428-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:41:17.634061 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-fe358061-27b8-4d97-b7f9-c2e77a64808c tempest-ServerRescueNegativeTestJSON-2101034428 tempest-ServerRescueNegativeTestJSON-2101034428-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:41:17.634461 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-fe358061-27b8-4d97-b7f9-c2e77a64808c tempest-ServerRescueNegativeTestJSON-2101034428 tempest-ServerRescueNegativeTestJSON-2101034428-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:41:17.635123 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-fe358061-27b8-4d97-b7f9-c2e77a64808c tempest-ServerRescueNegativeTestJSON-2101034428 tempest-ServerRescueNegativeTestJSON-2101034428-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:41:17.635538 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-fe358061-27b8-4d97-b7f9-c2e77a64808c tempest-ServerRescueNegativeTestJSON-2101034428 tempest-ServerRescueNegativeTestJSON-2101034428-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:41:17.635947 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-fe358061-27b8-4d97-b7f9-c2e77a64808c tempest-ServerRescueNegativeTestJSON-2101034428 tempest-ServerRescueNegativeTestJSON-2101034428-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:41:17.661266 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-fe358061-27b8-4d97-b7f9-c2e77a64808c tempest-ServerRescueNegativeTestJSON-2101034428 tempest-ServerRescueNegativeTestJSON-2101034428-project] Getting quotas for project c3a16922c1744d71bf7f99aa006f35e7. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 18 22:41:17.666789 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-fe358061-27b8-4d97-b7f9-c2e77a64808c tempest-ServerRescueNegativeTestJSON-2101034428 tempest-ServerRescueNegativeTestJSON-2101034428-project] Getting quotas for user 3c90eef5582b416d929dbafbd4442f34 and project c3a16922c1744d71bf7f99aa006f35e7. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 18 22:41:17.682448 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-fe358061-27b8-4d97-b7f9-c2e77a64808c tempest-ServerRescueNegativeTestJSON-2101034428 tempest-ServerRescueNegativeTestJSON-2101034428-project] [instance: 0b8c9848-73e6-4deb-b3c0-4ca1b0e58bd9] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120658) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 18 22:41:17.683253 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-fe358061-27b8-4d97-b7f9-c2e77a64808c tempest-ServerRescueNegativeTestJSON-2101034428 tempest-ServerRescueNegativeTestJSON-2101034428-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:41:17.683886 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-fe358061-27b8-4d97-b7f9-c2e77a64808c tempest-ServerRescueNegativeTestJSON-2101034428 tempest-ServerRescueNegativeTestJSON-2101034428-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:41:17.684291 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-fe358061-27b8-4d97-b7f9-c2e77a64808c tempest-ServerRescueNegativeTestJSON-2101034428 tempest-ServerRescueNegativeTestJSON-2101034428-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:41:17.732228 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-fe358061-27b8-4d97-b7f9-c2e77a64808c tempest-ServerRescueNegativeTestJSON-2101034428 tempest-ServerRescueNegativeTestJSON-2101034428-project] [instance: 0b8c9848-73e6-4deb-b3c0-4ca1b0e58bd9] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='87b0fec9-a948-41f9-a3c0-f0a6a5a8f1e7',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120658) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 18 22:41:17.733857 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-fe358061-27b8-4d97-b7f9-c2e77a64808c tempest-ServerRescueNegativeTestJSON-2101034428 tempest-ServerRescueNegativeTestJSON-2101034428-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:41:17.734294 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-fe358061-27b8-4d97-b7f9-c2e77a64808c tempest-ServerRescueNegativeTestJSON-2101034428 tempest-ServerRescueNegativeTestJSON-2101034428-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:41:17.734683 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-fe358061-27b8-4d97-b7f9-c2e77a64808c tempest-ServerRescueNegativeTestJSON-2101034428 tempest-ServerRescueNegativeTestJSON-2101034428-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:41:17.908623 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-fe358061-27b8-4d97-b7f9-c2e77a64808c tempest-ServerRescueNegativeTestJSON-2101034428 tempest-ServerRescueNegativeTestJSON-2101034428-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:41:17.909059 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-fe358061-27b8-4d97-b7f9-c2e77a64808c tempest-ServerRescueNegativeTestJSON-2101034428 tempest-ServerRescueNegativeTestJSON-2101034428-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:41:17.909451 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-fe358061-27b8-4d97-b7f9-c2e77a64808c tempest-ServerRescueNegativeTestJSON-2101034428 tempest-ServerRescueNegativeTestJSON-2101034428-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:41:17.909888 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-fe358061-27b8-4d97-b7f9-c2e77a64808c tempest-ServerRescueNegativeTestJSON-2101034428 tempest-ServerRescueNegativeTestJSON-2101034428-project] Acquiring lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 18 22:41:17.910405 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-fe358061-27b8-4d97-b7f9-c2e77a64808c tempest-ServerRescueNegativeTestJSON-2101034428 tempest-ServerRescueNegativeTestJSON-2101034428-project] Acquired lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 18 22:41:17.911127 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-e5309eb9-2806-4920-ba6f-4b5725c84705 None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:41:17.911802 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-e5309eb9-2806-4920-ba6f-4b5725c84705 None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:41:17.912119 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-e5309eb9-2806-4920-ba6f-4b5725c84705 None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:41:17.912759 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-e5309eb9-2806-4920-ba6f-4b5725c84705 None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:41:17.913344 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-e5309eb9-2806-4920-ba6f-4b5725c84705 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:41:17.913716 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-e5309eb9-2806-4920-ba6f-4b5725c84705 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:41:17.922628 n-d-853704-1 nova-conductor[120658]: ERROR nova.compute.rpcapi [None req-e5309eb9-2806-4920-ba6f-4b5725c84705 None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 18 22:41:17.923008 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-e5309eb9-2806-4920-ba6f-4b5725c84705 None None] Releasing lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 18 22:41:17.923945 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server [None req-e5309eb9-2806-4920-ba6f-4b5725c84705 None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:41:17.923945 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:41:17.923945 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 18 22:41:17.923945 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 18 22:41:17.923945 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 18 22:41:17.923945 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 22:41:17.923945 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 18 22:41:17.923945 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 22:41:17.923945 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:41:17.923945 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 18 22:41:17.923945 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 18 22:41:17.923945 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 18 22:41:17.923945 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 18 22:41:17.923945 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 18 22:41:17.923945 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 18 22:41:17.923945 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 18 22:41:17.923945 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 18 22:41:17.923945 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 18 22:41:17.923945 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 18 22:41:17.923945 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 18 22:41:17.923945 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 18 22:41:17.923945 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 18 22:41:17.923945 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 18 22:41:17.923945 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:41:17.923945 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 22:41:26.920540 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=14 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:41:26.921587 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=10,nova_api:UPDATE=1,nova_api:DELETE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:41:27.031140 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:41:27.031864 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell1:SELECT=11,nova_cell1:SAVEPOINT=1,nova_cell1:INSERT=18,nova_cell1:RELEASE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:41:27.796666 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=13,nova_api:UPDATE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:41:27.846942 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=11,nova_api:DELETE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:41:27.921792 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:41:27.922876 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell1:SELECT=11,nova_cell1:INSERT=17 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:41:43.153320 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=15,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:41:53.170139 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=12,nova_cell0:UPDATE=3 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:42:23.163400 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:42:33.186136 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:42:53.183170 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=3,nova_cell0:UPDATE=2 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:43:03.168540 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=7,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:43:30.328419 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-70a301c0-5ae1-4f1a-93e2-7fbd99adbf99 tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Took 0.75 seconds to select destinations for 1 instance(s). {{(pid=120657) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 18 22:43:30.350168 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-70a301c0-5ae1-4f1a-93e2-7fbd99adbf99 tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:43:30.350613 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-70a301c0-5ae1-4f1a-93e2-7fbd99adbf99 tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:43:30.351013 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-70a301c0-5ae1-4f1a-93e2-7fbd99adbf99 tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:43:30.534242 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-70a301c0-5ae1-4f1a-93e2-7fbd99adbf99 tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:43:30.534840 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-70a301c0-5ae1-4f1a-93e2-7fbd99adbf99 tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:43:30.535401 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-70a301c0-5ae1-4f1a-93e2-7fbd99adbf99 tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:43:30.536592 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-70a301c0-5ae1-4f1a-93e2-7fbd99adbf99 tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:43:30.537009 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-70a301c0-5ae1-4f1a-93e2-7fbd99adbf99 tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:43:30.537393 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-70a301c0-5ae1-4f1a-93e2-7fbd99adbf99 tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:43:30.556575 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-70a301c0-5ae1-4f1a-93e2-7fbd99adbf99 tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Getting quotas for project 59dde594f2d04d40886a969d353b0bc5. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 18 22:43:30.561373 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-70a301c0-5ae1-4f1a-93e2-7fbd99adbf99 tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Getting quotas for user 372a102cdb374e8e8d8f0ebe31c78e58 and project 59dde594f2d04d40886a969d353b0bc5. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 18 22:43:30.572438 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-70a301c0-5ae1-4f1a-93e2-7fbd99adbf99 tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] [instance: 55231079-4ce9-4c43-91f0-7525b1c49ebb] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120657) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 18 22:43:30.573276 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-70a301c0-5ae1-4f1a-93e2-7fbd99adbf99 tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:43:30.573686 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-70a301c0-5ae1-4f1a-93e2-7fbd99adbf99 tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:43:30.574072 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-70a301c0-5ae1-4f1a-93e2-7fbd99adbf99 tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:43:30.625843 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-70a301c0-5ae1-4f1a-93e2-7fbd99adbf99 tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] [instance: 55231079-4ce9-4c43-91f0-7525b1c49ebb] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='87b0fec9-a948-41f9-a3c0-f0a6a5a8f1e7',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120657) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 18 22:43:30.627059 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-70a301c0-5ae1-4f1a-93e2-7fbd99adbf99 tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:43:30.627492 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-70a301c0-5ae1-4f1a-93e2-7fbd99adbf99 tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:43:30.627922 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-70a301c0-5ae1-4f1a-93e2-7fbd99adbf99 tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:43:30.896356 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-70a301c0-5ae1-4f1a-93e2-7fbd99adbf99 tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:43:30.897061 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-70a301c0-5ae1-4f1a-93e2-7fbd99adbf99 tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:43:30.898286 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-70a301c0-5ae1-4f1a-93e2-7fbd99adbf99 tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:43:30.899053 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-70a301c0-5ae1-4f1a-93e2-7fbd99adbf99 tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Acquiring lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 18 22:43:30.900651 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-70a301c0-5ae1-4f1a-93e2-7fbd99adbf99 tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Acquired lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 18 22:43:30.902071 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-9cb2d27e-0ac7-43e0-86b6-57d6b1e0943e None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:43:30.902809 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-9cb2d27e-0ac7-43e0-86b6-57d6b1e0943e None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:43:30.903391 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-9cb2d27e-0ac7-43e0-86b6-57d6b1e0943e None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:43:30.904300 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-9cb2d27e-0ac7-43e0-86b6-57d6b1e0943e None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:43:30.905342 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-9cb2d27e-0ac7-43e0-86b6-57d6b1e0943e None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:43:30.906149 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-9cb2d27e-0ac7-43e0-86b6-57d6b1e0943e None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:43:30.916566 n-d-853704-1 nova-conductor[120657]: ERROR nova.compute.rpcapi [None req-9cb2d27e-0ac7-43e0-86b6-57d6b1e0943e None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 18 22:43:30.917854 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-9cb2d27e-0ac7-43e0-86b6-57d6b1e0943e None None] Releasing lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 18 22:43:30.918775 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server [None req-9cb2d27e-0ac7-43e0-86b6-57d6b1e0943e None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:43:30.918775 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:43:30.918775 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 18 22:43:30.918775 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 18 22:43:30.918775 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 18 22:43:30.918775 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 22:43:30.918775 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 18 22:43:30.918775 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 22:43:30.918775 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:43:30.918775 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 18 22:43:30.918775 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 18 22:43:30.918775 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 18 22:43:30.918775 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 18 22:43:30.918775 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 18 22:43:30.918775 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 18 22:43:30.918775 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 18 22:43:30.918775 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 18 22:43:30.918775 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 18 22:43:30.918775 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 18 22:43:30.918775 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 18 22:43:30.918775 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 18 22:43:30.918775 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 18 22:43:30.918775 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 18 22:43:30.918775 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:43:30.918775 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 22:43:33.193046 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:43:40.798794 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=12,nova_api:UPDATE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:43:40.799656 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=12,nova_api:DELETE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:43:40.914485 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:43:40.915517 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell1:SELECT=11,nova_cell1:INSERT=17 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:43:43.170585 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=14,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:44:13.200250 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=7,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:44:23.183593 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:44:32.653870 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-488434ee-a1db-4fb6-b0c0-30b6e6706d87 tempest-AttachVolumeTestJSON-1020115477 tempest-AttachVolumeTestJSON-1020115477-project] Took 0.91 seconds to select destinations for 1 instance(s). {{(pid=120658) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 18 22:44:32.676416 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-488434ee-a1db-4fb6-b0c0-30b6e6706d87 tempest-AttachVolumeTestJSON-1020115477 tempest-AttachVolumeTestJSON-1020115477-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:44:32.679834 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-488434ee-a1db-4fb6-b0c0-30b6e6706d87 tempest-AttachVolumeTestJSON-1020115477 tempest-AttachVolumeTestJSON-1020115477-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:44:32.679834 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-488434ee-a1db-4fb6-b0c0-30b6e6706d87 tempest-AttachVolumeTestJSON-1020115477 tempest-AttachVolumeTestJSON-1020115477-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:44:32.975959 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-488434ee-a1db-4fb6-b0c0-30b6e6706d87 tempest-AttachVolumeTestJSON-1020115477 tempest-AttachVolumeTestJSON-1020115477-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:44:32.976402 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-488434ee-a1db-4fb6-b0c0-30b6e6706d87 tempest-AttachVolumeTestJSON-1020115477 tempest-AttachVolumeTestJSON-1020115477-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:44:32.976794 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-488434ee-a1db-4fb6-b0c0-30b6e6706d87 tempest-AttachVolumeTestJSON-1020115477 tempest-AttachVolumeTestJSON-1020115477-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:44:32.977456 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-488434ee-a1db-4fb6-b0c0-30b6e6706d87 tempest-AttachVolumeTestJSON-1020115477 tempest-AttachVolumeTestJSON-1020115477-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:44:32.977913 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-488434ee-a1db-4fb6-b0c0-30b6e6706d87 tempest-AttachVolumeTestJSON-1020115477 tempest-AttachVolumeTestJSON-1020115477-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:44:32.978344 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-488434ee-a1db-4fb6-b0c0-30b6e6706d87 tempest-AttachVolumeTestJSON-1020115477 tempest-AttachVolumeTestJSON-1020115477-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:44:32.994883 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-488434ee-a1db-4fb6-b0c0-30b6e6706d87 tempest-AttachVolumeTestJSON-1020115477 tempest-AttachVolumeTestJSON-1020115477-project] Getting quotas for project 87577a6536624410a9ddb2715449de26. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 18 22:44:32.999705 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-488434ee-a1db-4fb6-b0c0-30b6e6706d87 tempest-AttachVolumeTestJSON-1020115477 tempest-AttachVolumeTestJSON-1020115477-project] Getting quotas for user d76c592c27574213aaf9a87e877f2016 and project 87577a6536624410a9ddb2715449de26. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 18 22:44:33.012081 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-488434ee-a1db-4fb6-b0c0-30b6e6706d87 tempest-AttachVolumeTestJSON-1020115477 tempest-AttachVolumeTestJSON-1020115477-project] [instance: 26aad67e-d3f3-49f9-9df4-05414f4d27ed] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120658) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 18 22:44:33.012866 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-488434ee-a1db-4fb6-b0c0-30b6e6706d87 tempest-AttachVolumeTestJSON-1020115477 tempest-AttachVolumeTestJSON-1020115477-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:44:33.013298 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-488434ee-a1db-4fb6-b0c0-30b6e6706d87 tempest-AttachVolumeTestJSON-1020115477 tempest-AttachVolumeTestJSON-1020115477-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:44:33.013686 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-488434ee-a1db-4fb6-b0c0-30b6e6706d87 tempest-AttachVolumeTestJSON-1020115477 tempest-AttachVolumeTestJSON-1020115477-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:44:33.074944 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-488434ee-a1db-4fb6-b0c0-30b6e6706d87 tempest-AttachVolumeTestJSON-1020115477 tempest-AttachVolumeTestJSON-1020115477-project] [instance: 26aad67e-d3f3-49f9-9df4-05414f4d27ed] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='87b0fec9-a948-41f9-a3c0-f0a6a5a8f1e7',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120658) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 18 22:44:33.076272 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-488434ee-a1db-4fb6-b0c0-30b6e6706d87 tempest-AttachVolumeTestJSON-1020115477 tempest-AttachVolumeTestJSON-1020115477-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:44:33.076794 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-488434ee-a1db-4fb6-b0c0-30b6e6706d87 tempest-AttachVolumeTestJSON-1020115477 tempest-AttachVolumeTestJSON-1020115477-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:44:33.077528 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-488434ee-a1db-4fb6-b0c0-30b6e6706d87 tempest-AttachVolumeTestJSON-1020115477 tempest-AttachVolumeTestJSON-1020115477-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:44:33.274730 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-488434ee-a1db-4fb6-b0c0-30b6e6706d87 tempest-AttachVolumeTestJSON-1020115477 tempest-AttachVolumeTestJSON-1020115477-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:44:33.275804 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-488434ee-a1db-4fb6-b0c0-30b6e6706d87 tempest-AttachVolumeTestJSON-1020115477 tempest-AttachVolumeTestJSON-1020115477-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:44:33.276435 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-488434ee-a1db-4fb6-b0c0-30b6e6706d87 tempest-AttachVolumeTestJSON-1020115477 tempest-AttachVolumeTestJSON-1020115477-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:44:33.277046 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-488434ee-a1db-4fb6-b0c0-30b6e6706d87 tempest-AttachVolumeTestJSON-1020115477 tempest-AttachVolumeTestJSON-1020115477-project] Acquiring lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 18 22:44:33.277804 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-488434ee-a1db-4fb6-b0c0-30b6e6706d87 tempest-AttachVolumeTestJSON-1020115477 tempest-AttachVolumeTestJSON-1020115477-project] Acquired lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 18 22:44:33.279082 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-216bb50f-d666-4f36-9bda-558fcde8cd00 None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:44:33.279818 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-216bb50f-d666-4f36-9bda-558fcde8cd00 None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:44:33.280552 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-216bb50f-d666-4f36-9bda-558fcde8cd00 None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:44:33.281691 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-216bb50f-d666-4f36-9bda-558fcde8cd00 None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:44:33.282277 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-216bb50f-d666-4f36-9bda-558fcde8cd00 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:44:33.282708 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-216bb50f-d666-4f36-9bda-558fcde8cd00 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:44:33.292981 n-d-853704-1 nova-conductor[120658]: ERROR nova.compute.rpcapi [None req-216bb50f-d666-4f36-9bda-558fcde8cd00 None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 18 22:44:33.293583 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-216bb50f-d666-4f36-9bda-558fcde8cd00 None None] Releasing lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 18 22:44:33.294916 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server [None req-216bb50f-d666-4f36-9bda-558fcde8cd00 None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:44:33.294916 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:44:33.294916 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 18 22:44:33.294916 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 18 22:44:33.294916 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 18 22:44:33.294916 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 22:44:33.294916 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 18 22:44:33.294916 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 22:44:33.294916 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:44:33.294916 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 18 22:44:33.294916 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 18 22:44:33.294916 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 18 22:44:33.294916 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 18 22:44:33.294916 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 18 22:44:33.294916 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 18 22:44:33.294916 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 18 22:44:33.294916 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 18 22:44:33.294916 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 18 22:44:33.294916 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 18 22:44:33.294916 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 18 22:44:33.294916 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 18 22:44:33.294916 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 18 22:44:33.294916 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 18 22:44:33.294916 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:44:33.294916 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 22:44:43.161678 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=12 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:44:43.219732 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=12,nova_api:UPDATE=1,nova_api:DELETE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:44:43.292949 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:44:43.294253 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell1:SELECT=11,nova_cell1:SAVEPOINT=1,nova_cell1:INSERT=18,nova_cell1:RELEASE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:44:53.201672 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=15,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:45:03.186769 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:45:23.194224 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=3,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:45:33.206105 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:45:41.335135 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-f3dd700f-906e-4ce8-820c-0b91ed98d77a tempest-ListImageFiltersTestJSON-2009003329 tempest-ListImageFiltersTestJSON-2009003329-project] Took 0.73 seconds to select destinations for 1 instance(s). {{(pid=120657) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 18 22:45:41.355435 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-f3dd700f-906e-4ce8-820c-0b91ed98d77a tempest-ListImageFiltersTestJSON-2009003329 tempest-ListImageFiltersTestJSON-2009003329-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:45:41.355938 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-f3dd700f-906e-4ce8-820c-0b91ed98d77a tempest-ListImageFiltersTestJSON-2009003329 tempest-ListImageFiltersTestJSON-2009003329-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:45:41.356330 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-f3dd700f-906e-4ce8-820c-0b91ed98d77a tempest-ListImageFiltersTestJSON-2009003329 tempest-ListImageFiltersTestJSON-2009003329-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:45:41.572863 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-f3dd700f-906e-4ce8-820c-0b91ed98d77a tempest-ListImageFiltersTestJSON-2009003329 tempest-ListImageFiltersTestJSON-2009003329-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:45:41.573383 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-f3dd700f-906e-4ce8-820c-0b91ed98d77a tempest-ListImageFiltersTestJSON-2009003329 tempest-ListImageFiltersTestJSON-2009003329-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:45:41.573993 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-f3dd700f-906e-4ce8-820c-0b91ed98d77a tempest-ListImageFiltersTestJSON-2009003329 tempest-ListImageFiltersTestJSON-2009003329-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:45:41.581281 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-f3dd700f-906e-4ce8-820c-0b91ed98d77a tempest-ListImageFiltersTestJSON-2009003329 tempest-ListImageFiltersTestJSON-2009003329-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:45:41.581281 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-f3dd700f-906e-4ce8-820c-0b91ed98d77a tempest-ListImageFiltersTestJSON-2009003329 tempest-ListImageFiltersTestJSON-2009003329-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:45:41.581281 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-f3dd700f-906e-4ce8-820c-0b91ed98d77a tempest-ListImageFiltersTestJSON-2009003329 tempest-ListImageFiltersTestJSON-2009003329-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:45:41.596870 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-f3dd700f-906e-4ce8-820c-0b91ed98d77a tempest-ListImageFiltersTestJSON-2009003329 tempest-ListImageFiltersTestJSON-2009003329-project] Getting quotas for project c63d878a2f8c4e3b81ab96c839989607. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 18 22:45:41.603569 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-f3dd700f-906e-4ce8-820c-0b91ed98d77a tempest-ListImageFiltersTestJSON-2009003329 tempest-ListImageFiltersTestJSON-2009003329-project] Getting quotas for user 5764ef52911e4de3a31c316c6c33c8ef and project c63d878a2f8c4e3b81ab96c839989607. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 18 22:45:41.617939 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-f3dd700f-906e-4ce8-820c-0b91ed98d77a tempest-ListImageFiltersTestJSON-2009003329 tempest-ListImageFiltersTestJSON-2009003329-project] [instance: 05d15f22-0b83-4e9f-ab98-53b4aca775c1] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120657) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 18 22:45:41.619145 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-f3dd700f-906e-4ce8-820c-0b91ed98d77a tempest-ListImageFiltersTestJSON-2009003329 tempest-ListImageFiltersTestJSON-2009003329-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:45:41.619587 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-f3dd700f-906e-4ce8-820c-0b91ed98d77a tempest-ListImageFiltersTestJSON-2009003329 tempest-ListImageFiltersTestJSON-2009003329-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:45:41.620062 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-f3dd700f-906e-4ce8-820c-0b91ed98d77a tempest-ListImageFiltersTestJSON-2009003329 tempest-ListImageFiltersTestJSON-2009003329-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:45:41.696251 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-f3dd700f-906e-4ce8-820c-0b91ed98d77a tempest-ListImageFiltersTestJSON-2009003329 tempest-ListImageFiltersTestJSON-2009003329-project] [instance: 05d15f22-0b83-4e9f-ab98-53b4aca775c1] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='87b0fec9-a948-41f9-a3c0-f0a6a5a8f1e7',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120657) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 18 22:45:41.697782 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-f3dd700f-906e-4ce8-820c-0b91ed98d77a tempest-ListImageFiltersTestJSON-2009003329 tempest-ListImageFiltersTestJSON-2009003329-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:45:41.698441 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-f3dd700f-906e-4ce8-820c-0b91ed98d77a tempest-ListImageFiltersTestJSON-2009003329 tempest-ListImageFiltersTestJSON-2009003329-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:45:41.698834 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-f3dd700f-906e-4ce8-820c-0b91ed98d77a tempest-ListImageFiltersTestJSON-2009003329 tempest-ListImageFiltersTestJSON-2009003329-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:45:42.053448 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-f3dd700f-906e-4ce8-820c-0b91ed98d77a tempest-ListImageFiltersTestJSON-2009003329 tempest-ListImageFiltersTestJSON-2009003329-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:45:42.054010 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-f3dd700f-906e-4ce8-820c-0b91ed98d77a tempest-ListImageFiltersTestJSON-2009003329 tempest-ListImageFiltersTestJSON-2009003329-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:45:42.054421 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-f3dd700f-906e-4ce8-820c-0b91ed98d77a tempest-ListImageFiltersTestJSON-2009003329 tempest-ListImageFiltersTestJSON-2009003329-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:45:42.054955 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-f3dd700f-906e-4ce8-820c-0b91ed98d77a tempest-ListImageFiltersTestJSON-2009003329 tempest-ListImageFiltersTestJSON-2009003329-project] Acquiring lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 18 22:45:42.055535 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-f3dd700f-906e-4ce8-820c-0b91ed98d77a tempest-ListImageFiltersTestJSON-2009003329 tempest-ListImageFiltersTestJSON-2009003329-project] Acquired lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 18 22:45:42.056554 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-d6faac14-ee45-46a3-b24e-e4546a483cef None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:45:42.057039 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-d6faac14-ee45-46a3-b24e-e4546a483cef None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:45:42.057492 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-d6faac14-ee45-46a3-b24e-e4546a483cef None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:45:42.059703 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-d6faac14-ee45-46a3-b24e-e4546a483cef None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:45:42.060138 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-d6faac14-ee45-46a3-b24e-e4546a483cef None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:45:42.060532 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-d6faac14-ee45-46a3-b24e-e4546a483cef None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:45:42.071934 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-029df9e7-f919-4f7c-94f2-c5d5fe471d7f tempest-ListImageFiltersTestJSON-2009003329 tempest-ListImageFiltersTestJSON-2009003329-project] Took 0.58 seconds to select destinations for 1 instance(s). {{(pid=120658) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 18 22:45:42.078221 n-d-853704-1 nova-conductor[120657]: ERROR nova.compute.rpcapi [None req-d6faac14-ee45-46a3-b24e-e4546a483cef None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 18 22:45:42.078611 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-d6faac14-ee45-46a3-b24e-e4546a483cef None None] Releasing lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 18 22:45:42.082721 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server [None req-d6faac14-ee45-46a3-b24e-e4546a483cef None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:45:42.082721 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:45:42.082721 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 18 22:45:42.082721 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 18 22:45:42.082721 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 18 22:45:42.082721 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 22:45:42.082721 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 18 22:45:42.082721 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 22:45:42.082721 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:45:42.082721 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 18 22:45:42.082721 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 18 22:45:42.082721 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 18 22:45:42.082721 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 18 22:45:42.082721 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 18 22:45:42.082721 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 18 22:45:42.082721 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 18 22:45:42.082721 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 18 22:45:42.082721 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 18 22:45:42.082721 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 18 22:45:42.082721 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 18 22:45:42.082721 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 18 22:45:42.082721 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 18 22:45:42.082721 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 18 22:45:42.082721 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:45:42.082721 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 22:45:42.098430 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-029df9e7-f919-4f7c-94f2-c5d5fe471d7f tempest-ListImageFiltersTestJSON-2009003329 tempest-ListImageFiltersTestJSON-2009003329-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:45:42.099046 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-029df9e7-f919-4f7c-94f2-c5d5fe471d7f tempest-ListImageFiltersTestJSON-2009003329 tempest-ListImageFiltersTestJSON-2009003329-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:45:42.099501 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-029df9e7-f919-4f7c-94f2-c5d5fe471d7f tempest-ListImageFiltersTestJSON-2009003329 tempest-ListImageFiltersTestJSON-2009003329-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:45:42.252998 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-029df9e7-f919-4f7c-94f2-c5d5fe471d7f tempest-ListImageFiltersTestJSON-2009003329 tempest-ListImageFiltersTestJSON-2009003329-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:45:42.253462 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-029df9e7-f919-4f7c-94f2-c5d5fe471d7f tempest-ListImageFiltersTestJSON-2009003329 tempest-ListImageFiltersTestJSON-2009003329-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:45:42.253849 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-029df9e7-f919-4f7c-94f2-c5d5fe471d7f tempest-ListImageFiltersTestJSON-2009003329 tempest-ListImageFiltersTestJSON-2009003329-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:45:42.254528 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-029df9e7-f919-4f7c-94f2-c5d5fe471d7f tempest-ListImageFiltersTestJSON-2009003329 tempest-ListImageFiltersTestJSON-2009003329-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:45:42.254932 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-029df9e7-f919-4f7c-94f2-c5d5fe471d7f tempest-ListImageFiltersTestJSON-2009003329 tempest-ListImageFiltersTestJSON-2009003329-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:45:42.255307 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-029df9e7-f919-4f7c-94f2-c5d5fe471d7f tempest-ListImageFiltersTestJSON-2009003329 tempest-ListImageFiltersTestJSON-2009003329-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:45:42.274515 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-029df9e7-f919-4f7c-94f2-c5d5fe471d7f tempest-ListImageFiltersTestJSON-2009003329 tempest-ListImageFiltersTestJSON-2009003329-project] Getting quotas for project c63d878a2f8c4e3b81ab96c839989607. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 18 22:45:42.279799 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-029df9e7-f919-4f7c-94f2-c5d5fe471d7f tempest-ListImageFiltersTestJSON-2009003329 tempest-ListImageFiltersTestJSON-2009003329-project] Getting quotas for user 5764ef52911e4de3a31c316c6c33c8ef and project c63d878a2f8c4e3b81ab96c839989607. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 18 22:45:42.290553 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-029df9e7-f919-4f7c-94f2-c5d5fe471d7f tempest-ListImageFiltersTestJSON-2009003329 tempest-ListImageFiltersTestJSON-2009003329-project] [instance: af31f77c-055f-41e5-acaa-ffb3a28064bf] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120658) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 18 22:45:42.291268 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-029df9e7-f919-4f7c-94f2-c5d5fe471d7f tempest-ListImageFiltersTestJSON-2009003329 tempest-ListImageFiltersTestJSON-2009003329-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:45:42.291718 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-029df9e7-f919-4f7c-94f2-c5d5fe471d7f tempest-ListImageFiltersTestJSON-2009003329 tempest-ListImageFiltersTestJSON-2009003329-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:45:42.296104 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-029df9e7-f919-4f7c-94f2-c5d5fe471d7f tempest-ListImageFiltersTestJSON-2009003329 tempest-ListImageFiltersTestJSON-2009003329-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:45:42.396158 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-029df9e7-f919-4f7c-94f2-c5d5fe471d7f tempest-ListImageFiltersTestJSON-2009003329 tempest-ListImageFiltersTestJSON-2009003329-project] [instance: af31f77c-055f-41e5-acaa-ffb3a28064bf] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='87b0fec9-a948-41f9-a3c0-f0a6a5a8f1e7',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120658) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 18 22:45:42.399407 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-029df9e7-f919-4f7c-94f2-c5d5fe471d7f tempest-ListImageFiltersTestJSON-2009003329 tempest-ListImageFiltersTestJSON-2009003329-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:45:42.399407 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-029df9e7-f919-4f7c-94f2-c5d5fe471d7f tempest-ListImageFiltersTestJSON-2009003329 tempest-ListImageFiltersTestJSON-2009003329-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:45:42.399407 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-029df9e7-f919-4f7c-94f2-c5d5fe471d7f tempest-ListImageFiltersTestJSON-2009003329 tempest-ListImageFiltersTestJSON-2009003329-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:45:42.795744 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-029df9e7-f919-4f7c-94f2-c5d5fe471d7f tempest-ListImageFiltersTestJSON-2009003329 tempest-ListImageFiltersTestJSON-2009003329-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:45:42.796432 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-029df9e7-f919-4f7c-94f2-c5d5fe471d7f tempest-ListImageFiltersTestJSON-2009003329 tempest-ListImageFiltersTestJSON-2009003329-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:45:42.797496 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-029df9e7-f919-4f7c-94f2-c5d5fe471d7f tempest-ListImageFiltersTestJSON-2009003329 tempest-ListImageFiltersTestJSON-2009003329-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:45:42.798340 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-029df9e7-f919-4f7c-94f2-c5d5fe471d7f tempest-ListImageFiltersTestJSON-2009003329 tempest-ListImageFiltersTestJSON-2009003329-project] Acquiring lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 18 22:45:42.798869 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-029df9e7-f919-4f7c-94f2-c5d5fe471d7f tempest-ListImageFiltersTestJSON-2009003329 tempest-ListImageFiltersTestJSON-2009003329-project] Acquired lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 18 22:45:42.799981 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-3757c39a-a6bf-4486-8ea2-655f84b83445 None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:45:42.800557 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-3757c39a-a6bf-4486-8ea2-655f84b83445 None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:45:42.801148 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-3757c39a-a6bf-4486-8ea2-655f84b83445 None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:45:42.803122 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-3757c39a-a6bf-4486-8ea2-655f84b83445 None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:45:42.803939 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-3757c39a-a6bf-4486-8ea2-655f84b83445 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.002s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:45:42.805689 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-3757c39a-a6bf-4486-8ea2-655f84b83445 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.002s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:45:42.830452 n-d-853704-1 nova-conductor[120658]: ERROR nova.compute.rpcapi [None req-3757c39a-a6bf-4486-8ea2-655f84b83445 None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 18 22:45:42.830847 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-3757c39a-a6bf-4486-8ea2-655f84b83445 None None] Releasing lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 18 22:45:42.832784 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server [None req-3757c39a-a6bf-4486-8ea2-655f84b83445 None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:45:42.832784 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:45:42.832784 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 18 22:45:42.832784 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 18 22:45:42.832784 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 18 22:45:42.832784 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 22:45:42.832784 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 18 22:45:42.832784 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 22:45:42.832784 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:45:42.832784 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 18 22:45:42.832784 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 18 22:45:42.832784 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 18 22:45:42.832784 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 18 22:45:42.832784 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 18 22:45:42.832784 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 18 22:45:42.832784 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 18 22:45:42.832784 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 18 22:45:42.832784 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 18 22:45:42.832784 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 18 22:45:42.832784 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 18 22:45:42.832784 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 18 22:45:42.832784 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 18 22:45:42.832784 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 18 22:45:42.832784 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:45:42.832784 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 22:45:51.796589 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=13,nova_api:UPDATE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:45:51.922451 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=11,nova_api:DELETE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:45:52.077906 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:45:52.078835 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell1:SELECT=11,nova_cell1:SAVEPOINT=1,nova_cell1:INSERT=18,nova_cell1:RELEASE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:45:52.667126 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=11,nova_api:UPDATE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:45:52.668015 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=13,nova_api:DELETE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:45:52.828158 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:45:52.829247 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell1:SELECT=11,nova_cell1:INSERT=17 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:45:53.202253 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=12,nova_cell0:UPDATE=3 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:46:13.203397 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=14,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:46:23.194418 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=6,nova_cell0:UPDATE=3 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:46:23.311178 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-5321f4ae-638c-46e4-9d67-644e881aeac2 tempest-TestNetworkBasicOps-944734306 tempest-TestNetworkBasicOps-944734306-project] Took 0.72 seconds to select destinations for 1 instance(s). {{(pid=120657) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 18 22:46:23.333090 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-5321f4ae-638c-46e4-9d67-644e881aeac2 tempest-TestNetworkBasicOps-944734306 tempest-TestNetworkBasicOps-944734306-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:46:23.333563 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-5321f4ae-638c-46e4-9d67-644e881aeac2 tempest-TestNetworkBasicOps-944734306 tempest-TestNetworkBasicOps-944734306-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:46:23.333965 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-5321f4ae-638c-46e4-9d67-644e881aeac2 tempest-TestNetworkBasicOps-944734306 tempest-TestNetworkBasicOps-944734306-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:46:23.521537 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-5321f4ae-638c-46e4-9d67-644e881aeac2 tempest-TestNetworkBasicOps-944734306 tempest-TestNetworkBasicOps-944734306-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:46:23.521974 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-5321f4ae-638c-46e4-9d67-644e881aeac2 tempest-TestNetworkBasicOps-944734306 tempest-TestNetworkBasicOps-944734306-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:46:23.522367 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-5321f4ae-638c-46e4-9d67-644e881aeac2 tempest-TestNetworkBasicOps-944734306 tempest-TestNetworkBasicOps-944734306-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:46:23.523041 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-5321f4ae-638c-46e4-9d67-644e881aeac2 tempest-TestNetworkBasicOps-944734306 tempest-TestNetworkBasicOps-944734306-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:46:23.523456 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-5321f4ae-638c-46e4-9d67-644e881aeac2 tempest-TestNetworkBasicOps-944734306 tempest-TestNetworkBasicOps-944734306-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:46:23.523873 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-5321f4ae-638c-46e4-9d67-644e881aeac2 tempest-TestNetworkBasicOps-944734306 tempest-TestNetworkBasicOps-944734306-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:46:23.541738 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-5321f4ae-638c-46e4-9d67-644e881aeac2 tempest-TestNetworkBasicOps-944734306 tempest-TestNetworkBasicOps-944734306-project] Getting quotas for project f0fb1cb4f0c54eb8906e4f1e8d0bf5d5. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 18 22:46:23.547428 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-5321f4ae-638c-46e4-9d67-644e881aeac2 tempest-TestNetworkBasicOps-944734306 tempest-TestNetworkBasicOps-944734306-project] Getting quotas for user 6460f85405fd4044bbb8edb65ad11f7a and project f0fb1cb4f0c54eb8906e4f1e8d0bf5d5. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 18 22:46:23.559654 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-5321f4ae-638c-46e4-9d67-644e881aeac2 tempest-TestNetworkBasicOps-944734306 tempest-TestNetworkBasicOps-944734306-project] [instance: 4e4b5e21-4752-4993-92b6-eeaed1ad9880] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120657) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 18 22:46:23.560500 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-5321f4ae-638c-46e4-9d67-644e881aeac2 tempest-TestNetworkBasicOps-944734306 tempest-TestNetworkBasicOps-944734306-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:46:23.560938 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-5321f4ae-638c-46e4-9d67-644e881aeac2 tempest-TestNetworkBasicOps-944734306 tempest-TestNetworkBasicOps-944734306-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:46:23.561332 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-5321f4ae-638c-46e4-9d67-644e881aeac2 tempest-TestNetworkBasicOps-944734306 tempest-TestNetworkBasicOps-944734306-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:46:23.629891 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-5321f4ae-638c-46e4-9d67-644e881aeac2 tempest-TestNetworkBasicOps-944734306 tempest-TestNetworkBasicOps-944734306-project] [instance: 4e4b5e21-4752-4993-92b6-eeaed1ad9880] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='87b0fec9-a948-41f9-a3c0-f0a6a5a8f1e7',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120657) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 18 22:46:23.631201 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-5321f4ae-638c-46e4-9d67-644e881aeac2 tempest-TestNetworkBasicOps-944734306 tempest-TestNetworkBasicOps-944734306-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:46:23.631726 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-5321f4ae-638c-46e4-9d67-644e881aeac2 tempest-TestNetworkBasicOps-944734306 tempest-TestNetworkBasicOps-944734306-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:46:23.632186 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-5321f4ae-638c-46e4-9d67-644e881aeac2 tempest-TestNetworkBasicOps-944734306 tempest-TestNetworkBasicOps-944734306-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:46:23.876874 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-5321f4ae-638c-46e4-9d67-644e881aeac2 tempest-TestNetworkBasicOps-944734306 tempest-TestNetworkBasicOps-944734306-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:46:23.880056 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-5321f4ae-638c-46e4-9d67-644e881aeac2 tempest-TestNetworkBasicOps-944734306 tempest-TestNetworkBasicOps-944734306-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:46:23.880056 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-5321f4ae-638c-46e4-9d67-644e881aeac2 tempest-TestNetworkBasicOps-944734306 tempest-TestNetworkBasicOps-944734306-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:46:23.880056 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-5321f4ae-638c-46e4-9d67-644e881aeac2 tempest-TestNetworkBasicOps-944734306 tempest-TestNetworkBasicOps-944734306-project] Acquiring lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 18 22:46:23.880056 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-5321f4ae-638c-46e4-9d67-644e881aeac2 tempest-TestNetworkBasicOps-944734306 tempest-TestNetworkBasicOps-944734306-project] Acquired lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 18 22:46:23.880056 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-96796d6d-48cc-4297-b998-99375b241e9c None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:46:23.880056 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-96796d6d-48cc-4297-b998-99375b241e9c None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:46:23.881135 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-96796d6d-48cc-4297-b998-99375b241e9c None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:46:23.882063 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-96796d6d-48cc-4297-b998-99375b241e9c None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:46:23.882650 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-96796d6d-48cc-4297-b998-99375b241e9c None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:46:23.883177 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-96796d6d-48cc-4297-b998-99375b241e9c None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:46:23.892921 n-d-853704-1 nova-conductor[120657]: ERROR nova.compute.rpcapi [None req-96796d6d-48cc-4297-b998-99375b241e9c None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 18 22:46:23.893305 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-96796d6d-48cc-4297-b998-99375b241e9c None None] Releasing lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 18 22:46:23.894219 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server [None req-96796d6d-48cc-4297-b998-99375b241e9c None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:46:23.894219 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:46:23.894219 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 18 22:46:23.894219 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 18 22:46:23.894219 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 18 22:46:23.894219 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 22:46:23.894219 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 18 22:46:23.894219 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 22:46:23.894219 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:46:23.894219 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 18 22:46:23.894219 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 18 22:46:23.894219 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 18 22:46:23.894219 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 18 22:46:23.894219 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 18 22:46:23.894219 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 18 22:46:23.894219 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 18 22:46:23.894219 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 18 22:46:23.894219 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 18 22:46:23.894219 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 18 22:46:23.894219 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 18 22:46:23.894219 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 18 22:46:23.894219 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 18 22:46:23.894219 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 18 22:46:23.894219 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:46:23.894219 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 22:46:33.709936 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=10 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:46:33.793794 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=14,nova_api:UPDATE=1,nova_api:DELETE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:46:33.892389 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:46:33.893057 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell1:SELECT=11,nova_cell1:SAVEPOINT=1,nova_cell1:INSERT=18,nova_cell1:RELEASE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:46:53.198278 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:47:03.186140 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=15,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:47:13.199829 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=3,nova_cell0:UPDATE=2 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:47:33.162553 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-2a21410a-ec0f-4718-92aa-ccbcd744aab6 tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Took 0.70 seconds to select destinations for 1 instance(s). {{(pid=120658) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 18 22:47:33.182805 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-2a21410a-ec0f-4718-92aa-ccbcd744aab6 tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:47:33.183331 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-2a21410a-ec0f-4718-92aa-ccbcd744aab6 tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:47:33.183812 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-2a21410a-ec0f-4718-92aa-ccbcd744aab6 tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:47:33.196269 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=5,nova_cell0:UPDATE=3 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:47:33.208610 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=4,nova_cell0:UPDATE=2 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:47:33.348615 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-2a21410a-ec0f-4718-92aa-ccbcd744aab6 tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:47:33.349055 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-2a21410a-ec0f-4718-92aa-ccbcd744aab6 tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:47:33.349450 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-2a21410a-ec0f-4718-92aa-ccbcd744aab6 tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:47:33.350115 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-2a21410a-ec0f-4718-92aa-ccbcd744aab6 tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:47:33.350524 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-2a21410a-ec0f-4718-92aa-ccbcd744aab6 tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:47:33.350905 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-2a21410a-ec0f-4718-92aa-ccbcd744aab6 tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:47:33.367049 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-2a21410a-ec0f-4718-92aa-ccbcd744aab6 tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Getting quotas for project 05baaf842a13444d8905683625e5853a. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 18 22:47:33.371804 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-2a21410a-ec0f-4718-92aa-ccbcd744aab6 tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Getting quotas for user 4d73c623dd454555ac8d974cf7a60398 and project 05baaf842a13444d8905683625e5853a. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 18 22:47:33.383785 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-2a21410a-ec0f-4718-92aa-ccbcd744aab6 tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] [instance: 665dfdbc-cf55-46a9-b342-92fe8f5acca0] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120658) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 18 22:47:33.384557 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-2a21410a-ec0f-4718-92aa-ccbcd744aab6 tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:47:33.384993 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-2a21410a-ec0f-4718-92aa-ccbcd744aab6 tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:47:33.385381 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-2a21410a-ec0f-4718-92aa-ccbcd744aab6 tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:47:33.491611 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-2a21410a-ec0f-4718-92aa-ccbcd744aab6 tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] [instance: 665dfdbc-cf55-46a9-b342-92fe8f5acca0] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='87b0fec9-a948-41f9-a3c0-f0a6a5a8f1e7',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120658) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 18 22:47:33.492931 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-2a21410a-ec0f-4718-92aa-ccbcd744aab6 tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:47:33.494152 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-2a21410a-ec0f-4718-92aa-ccbcd744aab6 tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:47:33.494692 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-2a21410a-ec0f-4718-92aa-ccbcd744aab6 tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:47:33.772638 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-2a21410a-ec0f-4718-92aa-ccbcd744aab6 tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:47:33.773073 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-2a21410a-ec0f-4718-92aa-ccbcd744aab6 tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:47:33.773467 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-2a21410a-ec0f-4718-92aa-ccbcd744aab6 tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:47:33.775857 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-2a21410a-ec0f-4718-92aa-ccbcd744aab6 tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Acquiring lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 18 22:47:33.776505 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-2a21410a-ec0f-4718-92aa-ccbcd744aab6 tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Acquired lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 18 22:47:33.778201 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ee423054-03bb-4489-9723-98a55c25f75f None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:47:33.778899 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ee423054-03bb-4489-9723-98a55c25f75f None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:47:33.780056 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ee423054-03bb-4489-9723-98a55c25f75f None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:47:33.781614 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ee423054-03bb-4489-9723-98a55c25f75f None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:47:33.783767 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ee423054-03bb-4489-9723-98a55c25f75f None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:47:33.786263 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ee423054-03bb-4489-9723-98a55c25f75f None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.004s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:47:33.806501 n-d-853704-1 nova-conductor[120658]: ERROR nova.compute.rpcapi [None req-ee423054-03bb-4489-9723-98a55c25f75f None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 18 22:47:33.806501 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ee423054-03bb-4489-9723-98a55c25f75f None None] Releasing lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 18 22:47:33.806501 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server [None req-ee423054-03bb-4489-9723-98a55c25f75f None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:47:33.806501 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:47:33.806501 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 18 22:47:33.806501 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 18 22:47:33.806501 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 18 22:47:33.806501 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 22:47:33.806501 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 18 22:47:33.806501 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 22:47:33.806501 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:47:33.806501 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 18 22:47:33.806501 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 18 22:47:33.806501 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 18 22:47:33.806501 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 18 22:47:33.806501 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 18 22:47:33.806501 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 18 22:47:33.806501 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 18 22:47:33.806501 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 18 22:47:33.806501 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 18 22:47:33.806501 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 18 22:47:33.806501 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 18 22:47:33.806501 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 18 22:47:33.806501 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 18 22:47:33.806501 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 18 22:47:33.806501 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:47:33.806501 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 22:47:43.696371 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=11,nova_api:UPDATE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:47:43.698033 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=13,nova_api:DELETE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:47:43.803326 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:47:43.804120 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell1:SELECT=11,nova_cell1:SAVEPOINT=1,nova_cell1:INSERT=18,nova_cell1:RELEASE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:48:03.208529 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=6,nova_cell0:UPDATE=3 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:48:13.339741 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=15,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:48:33.215843 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=6,nova_cell0:UPDATE=3 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:48:53.212263 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:49:03.234306 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=6,nova_cell0:UPDATE=3 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:49:30.982780 n-d-853704-1 nova-conductor[120657]: ERROR oslo.messaging._drivers.impl_rabbit [None req-94022c29-e216-4ae4-800c-79bd68d6a844 tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] [ae71ad44-1ef2-45ec-82d0-4ccfc1149604] AMQP server on 10.222.0.49:5672 is unreachable: [Errno 104] Connection reset by peer. Trying again in 1 seconds.: ConnectionResetError: [Errno 104] Connection reset by peer Aug 18 22:49:31.991005 n-d-853704-1 nova-conductor[120657]: INFO oslo.messaging._drivers.impl_rabbit [None req-94022c29-e216-4ae4-800c-79bd68d6a844 tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] [ae71ad44-1ef2-45ec-82d0-4ccfc1149604] Reconnected to AMQP server on 10.222.0.49:5672 via [amqp] client with port 43468. Aug 18 22:49:33.215622 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=7,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:49:33.266070 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-bb2cee24-d894-41a6-9887-58f8c164a7dd tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Took 0.52 seconds to select destinations for 1 instance(s). {{(pid=120658) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 18 22:49:33.294641 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-bb2cee24-d894-41a6-9887-58f8c164a7dd tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:49:33.301508 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-bb2cee24-d894-41a6-9887-58f8c164a7dd tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:49:33.301508 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-bb2cee24-d894-41a6-9887-58f8c164a7dd tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:49:33.481688 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-bb2cee24-d894-41a6-9887-58f8c164a7dd tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:49:33.482133 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-bb2cee24-d894-41a6-9887-58f8c164a7dd tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:49:33.482526 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-bb2cee24-d894-41a6-9887-58f8c164a7dd tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:49:33.483190 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-bb2cee24-d894-41a6-9887-58f8c164a7dd tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:49:33.483649 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-bb2cee24-d894-41a6-9887-58f8c164a7dd tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:49:33.484074 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-bb2cee24-d894-41a6-9887-58f8c164a7dd tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:49:33.503361 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-bb2cee24-d894-41a6-9887-58f8c164a7dd tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Getting quotas for project 59dde594f2d04d40886a969d353b0bc5. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 18 22:49:33.509411 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-bb2cee24-d894-41a6-9887-58f8c164a7dd tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Getting quotas for user 372a102cdb374e8e8d8f0ebe31c78e58 and project 59dde594f2d04d40886a969d353b0bc5. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 18 22:49:33.521557 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-bb2cee24-d894-41a6-9887-58f8c164a7dd tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] [instance: 2eceb488-fb81-4b7c-b681-162eedf1f233] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120658) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 18 22:49:33.522437 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-bb2cee24-d894-41a6-9887-58f8c164a7dd tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:49:33.523440 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-bb2cee24-d894-41a6-9887-58f8c164a7dd tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:49:33.524447 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-bb2cee24-d894-41a6-9887-58f8c164a7dd tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:49:33.645953 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-bb2cee24-d894-41a6-9887-58f8c164a7dd tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] [instance: 2eceb488-fb81-4b7c-b681-162eedf1f233] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='87b0fec9-a948-41f9-a3c0-f0a6a5a8f1e7',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120658) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 18 22:49:33.647175 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-bb2cee24-d894-41a6-9887-58f8c164a7dd tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:49:33.647693 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-bb2cee24-d894-41a6-9887-58f8c164a7dd tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:49:33.648383 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-bb2cee24-d894-41a6-9887-58f8c164a7dd tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:49:33.962002 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-bb2cee24-d894-41a6-9887-58f8c164a7dd tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:49:33.962461 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-bb2cee24-d894-41a6-9887-58f8c164a7dd tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:49:33.962857 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-bb2cee24-d894-41a6-9887-58f8c164a7dd tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:49:33.963844 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-bb2cee24-d894-41a6-9887-58f8c164a7dd tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Acquiring lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 18 22:49:33.964389 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-bb2cee24-d894-41a6-9887-58f8c164a7dd tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Acquired lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 18 22:49:33.965133 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-15106fdb-9fc3-4db4-b772-31649c6c8fbc None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:49:33.965593 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-15106fdb-9fc3-4db4-b772-31649c6c8fbc None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:49:33.965964 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-15106fdb-9fc3-4db4-b772-31649c6c8fbc None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:49:33.966617 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-15106fdb-9fc3-4db4-b772-31649c6c8fbc None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:49:33.967157 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-15106fdb-9fc3-4db4-b772-31649c6c8fbc None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:49:33.967569 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-15106fdb-9fc3-4db4-b772-31649c6c8fbc None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:49:33.977317 n-d-853704-1 nova-conductor[120658]: ERROR nova.compute.rpcapi [None req-15106fdb-9fc3-4db4-b772-31649c6c8fbc None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 18 22:49:33.987156 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-15106fdb-9fc3-4db4-b772-31649c6c8fbc None None] Releasing lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 18 22:49:33.987156 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server [None req-15106fdb-9fc3-4db4-b772-31649c6c8fbc None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:49:33.987156 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:49:33.987156 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 18 22:49:33.987156 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 18 22:49:33.987156 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 18 22:49:33.987156 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 22:49:33.987156 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 18 22:49:33.987156 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 22:49:33.987156 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:49:33.987156 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 18 22:49:33.987156 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 18 22:49:33.987156 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 18 22:49:33.987156 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 18 22:49:33.987156 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 18 22:49:33.987156 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 18 22:49:33.987156 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 18 22:49:33.987156 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 18 22:49:33.987156 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 18 22:49:33.987156 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 18 22:49:33.987156 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 18 22:49:33.987156 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 18 22:49:33.987156 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 18 22:49:33.987156 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 18 22:49:33.987156 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:49:33.987156 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 22:49:34.590524 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-94022c29-e216-4ae4-800c-79bd68d6a844 tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Took 3.61 seconds to select destinations for 1 instance(s). {{(pid=120657) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 18 22:49:34.603641 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-94022c29-e216-4ae4-800c-79bd68d6a844 tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] [instance: 59395d53-81a3-4c56-b4b5-a3f28da4ffec] While scheduling instance, the build request was already deleted. {{(pid=120657) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1606}} Aug 18 22:49:34.979704 n-d-853704-1 nova-conductor[120657]: INFO nova.scheduler.client.report [None req-94022c29-e216-4ae4-800c-79bd68d6a844 tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Deleted allocations for instance 59395d53-81a3-4c56-b4b5-a3f28da4ffec Aug 18 22:49:35.003194 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:49:35.007253 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-94022c29-e216-4ae4-800c-79bd68d6a844 tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:49:35.007860 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-94022c29-e216-4ae4-800c-79bd68d6a844 tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:49:35.008344 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-94022c29-e216-4ae4-800c-79bd68d6a844 tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:49:35.009145 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-94022c29-e216-4ae4-800c-79bd68d6a844 tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:49:35.009594 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-94022c29-e216-4ae4-800c-79bd68d6a844 tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:49:35.010115 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-94022c29-e216-4ae4-800c-79bd68d6a844 tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:49:35.030309 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-94022c29-e216-4ae4-800c-79bd68d6a844 tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Getting quotas for project 59dde594f2d04d40886a969d353b0bc5. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 18 22:49:35.035419 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-94022c29-e216-4ae4-800c-79bd68d6a844 tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Getting quotas for user 372a102cdb374e8e8d8f0ebe31c78e58 and project 59dde594f2d04d40886a969d353b0bc5. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 18 22:49:43.860195 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=17,nova_api:UPDATE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:49:43.861669 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=7,nova_api:DELETE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:49:43.975399 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:49:43.976414 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell1:SELECT=11,nova_cell1:INSERT=17 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:49:45.020146 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:49:45.022242 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell1:SELECT=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:49:45.038537 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=14 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:49:45.049526 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=5,nova_api:DELETE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:49:53.217728 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=10,nova_cell0:UPDATE=2 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:50:13.222653 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=12,nova_cell0:UPDATE=3 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:50:33.222015 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:50:38.138782 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-dd42f901-5701-4aeb-a91a-c9b68e71fa47 tempest-AttachVolumeTestJSON-1020115477 tempest-AttachVolumeTestJSON-1020115477-project] Took 0.43 seconds to select destinations for 1 instance(s). {{(pid=120657) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 18 22:50:38.160542 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-dd42f901-5701-4aeb-a91a-c9b68e71fa47 tempest-AttachVolumeTestJSON-1020115477 tempest-AttachVolumeTestJSON-1020115477-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:50:38.160982 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-dd42f901-5701-4aeb-a91a-c9b68e71fa47 tempest-AttachVolumeTestJSON-1020115477 tempest-AttachVolumeTestJSON-1020115477-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:50:38.161372 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-dd42f901-5701-4aeb-a91a-c9b68e71fa47 tempest-AttachVolumeTestJSON-1020115477 tempest-AttachVolumeTestJSON-1020115477-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:50:38.254815 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-dd42f901-5701-4aeb-a91a-c9b68e71fa47 tempest-AttachVolumeTestJSON-1020115477 tempest-AttachVolumeTestJSON-1020115477-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:50:38.255260 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-dd42f901-5701-4aeb-a91a-c9b68e71fa47 tempest-AttachVolumeTestJSON-1020115477 tempest-AttachVolumeTestJSON-1020115477-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:50:38.255652 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-dd42f901-5701-4aeb-a91a-c9b68e71fa47 tempest-AttachVolumeTestJSON-1020115477 tempest-AttachVolumeTestJSON-1020115477-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:50:38.256341 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-dd42f901-5701-4aeb-a91a-c9b68e71fa47 tempest-AttachVolumeTestJSON-1020115477 tempest-AttachVolumeTestJSON-1020115477-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:50:38.256756 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-dd42f901-5701-4aeb-a91a-c9b68e71fa47 tempest-AttachVolumeTestJSON-1020115477 tempest-AttachVolumeTestJSON-1020115477-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:50:38.257139 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-dd42f901-5701-4aeb-a91a-c9b68e71fa47 tempest-AttachVolumeTestJSON-1020115477 tempest-AttachVolumeTestJSON-1020115477-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:50:38.272646 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-dd42f901-5701-4aeb-a91a-c9b68e71fa47 tempest-AttachVolumeTestJSON-1020115477 tempest-AttachVolumeTestJSON-1020115477-project] Getting quotas for project 87577a6536624410a9ddb2715449de26. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 18 22:50:38.277498 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-dd42f901-5701-4aeb-a91a-c9b68e71fa47 tempest-AttachVolumeTestJSON-1020115477 tempest-AttachVolumeTestJSON-1020115477-project] Getting quotas for user d76c592c27574213aaf9a87e877f2016 and project 87577a6536624410a9ddb2715449de26. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 18 22:50:38.288691 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-dd42f901-5701-4aeb-a91a-c9b68e71fa47 tempest-AttachVolumeTestJSON-1020115477 tempest-AttachVolumeTestJSON-1020115477-project] [instance: 5562535a-532c-4663-b89a-a8a34f28bc6b] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120657) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 18 22:50:38.290001 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-dd42f901-5701-4aeb-a91a-c9b68e71fa47 tempest-AttachVolumeTestJSON-1020115477 tempest-AttachVolumeTestJSON-1020115477-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:50:38.290446 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-dd42f901-5701-4aeb-a91a-c9b68e71fa47 tempest-AttachVolumeTestJSON-1020115477 tempest-AttachVolumeTestJSON-1020115477-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:50:38.290839 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-dd42f901-5701-4aeb-a91a-c9b68e71fa47 tempest-AttachVolumeTestJSON-1020115477 tempest-AttachVolumeTestJSON-1020115477-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:50:38.340913 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-dd42f901-5701-4aeb-a91a-c9b68e71fa47 tempest-AttachVolumeTestJSON-1020115477 tempest-AttachVolumeTestJSON-1020115477-project] [instance: 5562535a-532c-4663-b89a-a8a34f28bc6b] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='87b0fec9-a948-41f9-a3c0-f0a6a5a8f1e7',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120657) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 18 22:50:38.342154 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-dd42f901-5701-4aeb-a91a-c9b68e71fa47 tempest-AttachVolumeTestJSON-1020115477 tempest-AttachVolumeTestJSON-1020115477-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:50:38.342597 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-dd42f901-5701-4aeb-a91a-c9b68e71fa47 tempest-AttachVolumeTestJSON-1020115477 tempest-AttachVolumeTestJSON-1020115477-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:50:38.342999 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-dd42f901-5701-4aeb-a91a-c9b68e71fa47 tempest-AttachVolumeTestJSON-1020115477 tempest-AttachVolumeTestJSON-1020115477-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:50:38.589428 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-dd42f901-5701-4aeb-a91a-c9b68e71fa47 tempest-AttachVolumeTestJSON-1020115477 tempest-AttachVolumeTestJSON-1020115477-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:50:38.589775 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-dd42f901-5701-4aeb-a91a-c9b68e71fa47 tempest-AttachVolumeTestJSON-1020115477 tempest-AttachVolumeTestJSON-1020115477-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:50:38.590190 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-dd42f901-5701-4aeb-a91a-c9b68e71fa47 tempest-AttachVolumeTestJSON-1020115477 tempest-AttachVolumeTestJSON-1020115477-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:50:38.590666 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-dd42f901-5701-4aeb-a91a-c9b68e71fa47 tempest-AttachVolumeTestJSON-1020115477 tempest-AttachVolumeTestJSON-1020115477-project] Acquiring lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 18 22:50:38.591014 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-dd42f901-5701-4aeb-a91a-c9b68e71fa47 tempest-AttachVolumeTestJSON-1020115477 tempest-AttachVolumeTestJSON-1020115477-project] Acquired lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 18 22:50:38.591800 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-b640eff1-988a-45d1-9bcc-821e88e38665 None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:50:38.592266 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-b640eff1-988a-45d1-9bcc-821e88e38665 None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:50:38.592654 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-b640eff1-988a-45d1-9bcc-821e88e38665 None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:50:38.593319 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-b640eff1-988a-45d1-9bcc-821e88e38665 None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:50:38.593729 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-b640eff1-988a-45d1-9bcc-821e88e38665 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:50:38.594107 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-b640eff1-988a-45d1-9bcc-821e88e38665 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:50:38.602337 n-d-853704-1 nova-conductor[120657]: ERROR nova.compute.rpcapi [None req-b640eff1-988a-45d1-9bcc-821e88e38665 None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 18 22:50:38.602731 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-b640eff1-988a-45d1-9bcc-821e88e38665 None None] Releasing lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 18 22:50:38.603626 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server [None req-b640eff1-988a-45d1-9bcc-821e88e38665 None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:50:38.603626 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:50:38.603626 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 18 22:50:38.603626 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 18 22:50:38.603626 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 18 22:50:38.603626 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 22:50:38.603626 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 18 22:50:38.603626 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 22:50:38.603626 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:50:38.603626 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 18 22:50:38.603626 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 18 22:50:38.603626 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 18 22:50:38.603626 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 18 22:50:38.603626 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 18 22:50:38.603626 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 18 22:50:38.603626 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 18 22:50:38.603626 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 18 22:50:38.603626 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 18 22:50:38.603626 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 18 22:50:38.603626 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 18 22:50:38.603626 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 18 22:50:38.603626 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 18 22:50:38.603626 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 18 22:50:38.603626 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:50:38.603626 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 22:50:48.430704 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=12 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:50:48.509793 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=12,nova_api:UPDATE=1,nova_api:DELETE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:50:48.600333 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:50:48.601030 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell1:SELECT=11,nova_cell1:INSERT=17 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:50:53.228145 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=14,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:51:13.231835 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:51:33.223963 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:51:36.530924 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-9ac9d435-148f-42c3-bce2-4a3ec655deda tempest-TestVolumeBackupRestore-1677249161 tempest-TestVolumeBackupRestore-1677249161-project] Took 0.55 seconds to select destinations for 1 instance(s). {{(pid=120658) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 18 22:51:36.551202 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-9ac9d435-148f-42c3-bce2-4a3ec655deda tempest-TestVolumeBackupRestore-1677249161 tempest-TestVolumeBackupRestore-1677249161-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:51:36.551647 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-9ac9d435-148f-42c3-bce2-4a3ec655deda tempest-TestVolumeBackupRestore-1677249161 tempest-TestVolumeBackupRestore-1677249161-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:51:36.552069 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-9ac9d435-148f-42c3-bce2-4a3ec655deda tempest-TestVolumeBackupRestore-1677249161 tempest-TestVolumeBackupRestore-1677249161-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:51:36.746341 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-9ac9d435-148f-42c3-bce2-4a3ec655deda tempest-TestVolumeBackupRestore-1677249161 tempest-TestVolumeBackupRestore-1677249161-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:51:36.746706 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-9ac9d435-148f-42c3-bce2-4a3ec655deda tempest-TestVolumeBackupRestore-1677249161 tempest-TestVolumeBackupRestore-1677249161-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:51:36.747079 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-9ac9d435-148f-42c3-bce2-4a3ec655deda tempest-TestVolumeBackupRestore-1677249161 tempest-TestVolumeBackupRestore-1677249161-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:51:36.747762 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-9ac9d435-148f-42c3-bce2-4a3ec655deda tempest-TestVolumeBackupRestore-1677249161 tempest-TestVolumeBackupRestore-1677249161-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:51:36.748174 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-9ac9d435-148f-42c3-bce2-4a3ec655deda tempest-TestVolumeBackupRestore-1677249161 tempest-TestVolumeBackupRestore-1677249161-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:51:36.748563 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-9ac9d435-148f-42c3-bce2-4a3ec655deda tempest-TestVolumeBackupRestore-1677249161 tempest-TestVolumeBackupRestore-1677249161-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:51:36.764703 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-9ac9d435-148f-42c3-bce2-4a3ec655deda tempest-TestVolumeBackupRestore-1677249161 tempest-TestVolumeBackupRestore-1677249161-project] Getting quotas for project 139c1e0bcbdd40f1afb85f0c45f65032. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 18 22:51:36.769837 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-9ac9d435-148f-42c3-bce2-4a3ec655deda tempest-TestVolumeBackupRestore-1677249161 tempest-TestVolumeBackupRestore-1677249161-project] Getting quotas for user d3624a6a26624c4c87fe8a69da2fd7b2 and project 139c1e0bcbdd40f1afb85f0c45f65032. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 18 22:51:36.780818 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-9ac9d435-148f-42c3-bce2-4a3ec655deda tempest-TestVolumeBackupRestore-1677249161 tempest-TestVolumeBackupRestore-1677249161-project] [instance: b0a56fcb-b16e-41e8-b48e-286061b796c5] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120658) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 18 22:51:36.782205 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-9ac9d435-148f-42c3-bce2-4a3ec655deda tempest-TestVolumeBackupRestore-1677249161 tempest-TestVolumeBackupRestore-1677249161-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:51:36.782767 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-9ac9d435-148f-42c3-bce2-4a3ec655deda tempest-TestVolumeBackupRestore-1677249161 tempest-TestVolumeBackupRestore-1677249161-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:51:36.783236 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-9ac9d435-148f-42c3-bce2-4a3ec655deda tempest-TestVolumeBackupRestore-1677249161 tempest-TestVolumeBackupRestore-1677249161-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:51:36.896769 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-9ac9d435-148f-42c3-bce2-4a3ec655deda tempest-TestVolumeBackupRestore-1677249161 tempest-TestVolumeBackupRestore-1677249161-project] [instance: b0a56fcb-b16e-41e8-b48e-286061b796c5] block_device_mapping [BlockDeviceMapping(attachment_id=d2034355-c41f-4084-b70b-4c921727ce14,boot_index=0,connection_info=None,created_at=,delete_on_termination=False,deleted=,deleted_at=,destination_type='volume',device_name=None,device_type=None,disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id=None,instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='volume',tag=None,updated_at=,uuid=,volume_id='467d2ea1-38c0-4ff1-b83e-e096ec8a749e',volume_size=1,volume_type=None)] {{(pid=120658) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 18 22:51:36.898027 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-9ac9d435-148f-42c3-bce2-4a3ec655deda tempest-TestVolumeBackupRestore-1677249161 tempest-TestVolumeBackupRestore-1677249161-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:51:36.898487 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-9ac9d435-148f-42c3-bce2-4a3ec655deda tempest-TestVolumeBackupRestore-1677249161 tempest-TestVolumeBackupRestore-1677249161-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:51:36.898875 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-9ac9d435-148f-42c3-bce2-4a3ec655deda tempest-TestVolumeBackupRestore-1677249161 tempest-TestVolumeBackupRestore-1677249161-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:51:37.130026 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-9ac9d435-148f-42c3-bce2-4a3ec655deda tempest-TestVolumeBackupRestore-1677249161 tempest-TestVolumeBackupRestore-1677249161-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:51:37.130548 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-9ac9d435-148f-42c3-bce2-4a3ec655deda tempest-TestVolumeBackupRestore-1677249161 tempest-TestVolumeBackupRestore-1677249161-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:51:37.131003 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-9ac9d435-148f-42c3-bce2-4a3ec655deda tempest-TestVolumeBackupRestore-1677249161 tempest-TestVolumeBackupRestore-1677249161-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:51:37.131529 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-9ac9d435-148f-42c3-bce2-4a3ec655deda tempest-TestVolumeBackupRestore-1677249161 tempest-TestVolumeBackupRestore-1677249161-project] Acquiring lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 18 22:51:37.131965 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-9ac9d435-148f-42c3-bce2-4a3ec655deda tempest-TestVolumeBackupRestore-1677249161 tempest-TestVolumeBackupRestore-1677249161-project] Acquired lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 18 22:51:37.132680 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-f0734050-1d7a-4125-a02d-de3e7fb1cf0b None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:51:37.133080 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-f0734050-1d7a-4125-a02d-de3e7fb1cf0b None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:51:37.133452 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-f0734050-1d7a-4125-a02d-de3e7fb1cf0b None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:51:37.134082 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-f0734050-1d7a-4125-a02d-de3e7fb1cf0b None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:51:37.134744 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-f0734050-1d7a-4125-a02d-de3e7fb1cf0b None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:51:37.135150 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-f0734050-1d7a-4125-a02d-de3e7fb1cf0b None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:51:37.144078 n-d-853704-1 nova-conductor[120658]: ERROR nova.compute.rpcapi [None req-f0734050-1d7a-4125-a02d-de3e7fb1cf0b None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 18 22:51:37.145199 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-f0734050-1d7a-4125-a02d-de3e7fb1cf0b None None] Releasing lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 18 22:51:37.145438 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server [None req-f0734050-1d7a-4125-a02d-de3e7fb1cf0b None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:51:37.145438 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:51:37.145438 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 18 22:51:37.145438 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 18 22:51:37.145438 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 18 22:51:37.145438 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 22:51:37.145438 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 18 22:51:37.145438 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 22:51:37.145438 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:51:37.145438 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 18 22:51:37.145438 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 18 22:51:37.145438 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 18 22:51:37.145438 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 18 22:51:37.145438 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 18 22:51:37.145438 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 18 22:51:37.145438 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 18 22:51:37.145438 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 18 22:51:37.145438 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 18 22:51:37.145438 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 18 22:51:37.145438 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 18 22:51:37.145438 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 18 22:51:37.145438 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 18 22:51:37.145438 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 18 22:51:37.145438 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:51:37.145438 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 22:51:46.992259 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=16 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:51:47.059567 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=8,nova_api:UPDATE=1,nova_api:DELETE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:51:47.141903 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:51:47.142711 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell1:SELECT=11,nova_cell1:SAVEPOINT=1,nova_cell1:INSERT=17,nova_cell1:RELEASE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:51:53.229901 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=13,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:51:53.230611 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=3,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:51:54.959612 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-56e48629-2ffc-4fe5-925f-4ac99458f3f4 tempest-InstanceActionsV221TestJSON-1507438675 tempest-InstanceActionsV221TestJSON-1507438675-project] Took 0.56 seconds to select destinations for 1 instance(s). {{(pid=120657) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 18 22:51:54.981114 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-56e48629-2ffc-4fe5-925f-4ac99458f3f4 tempest-InstanceActionsV221TestJSON-1507438675 tempest-InstanceActionsV221TestJSON-1507438675-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:51:54.981728 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-56e48629-2ffc-4fe5-925f-4ac99458f3f4 tempest-InstanceActionsV221TestJSON-1507438675 tempest-InstanceActionsV221TestJSON-1507438675-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:51:54.982326 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-56e48629-2ffc-4fe5-925f-4ac99458f3f4 tempest-InstanceActionsV221TestJSON-1507438675 tempest-InstanceActionsV221TestJSON-1507438675-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:51:55.169867 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-56e48629-2ffc-4fe5-925f-4ac99458f3f4 tempest-InstanceActionsV221TestJSON-1507438675 tempest-InstanceActionsV221TestJSON-1507438675-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:51:55.170223 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-56e48629-2ffc-4fe5-925f-4ac99458f3f4 tempest-InstanceActionsV221TestJSON-1507438675 tempest-InstanceActionsV221TestJSON-1507438675-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:51:55.170619 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-56e48629-2ffc-4fe5-925f-4ac99458f3f4 tempest-InstanceActionsV221TestJSON-1507438675 tempest-InstanceActionsV221TestJSON-1507438675-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:51:55.171286 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-56e48629-2ffc-4fe5-925f-4ac99458f3f4 tempest-InstanceActionsV221TestJSON-1507438675 tempest-InstanceActionsV221TestJSON-1507438675-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:51:55.171804 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-56e48629-2ffc-4fe5-925f-4ac99458f3f4 tempest-InstanceActionsV221TestJSON-1507438675 tempest-InstanceActionsV221TestJSON-1507438675-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:51:55.172222 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-56e48629-2ffc-4fe5-925f-4ac99458f3f4 tempest-InstanceActionsV221TestJSON-1507438675 tempest-InstanceActionsV221TestJSON-1507438675-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:51:55.187299 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-56e48629-2ffc-4fe5-925f-4ac99458f3f4 tempest-InstanceActionsV221TestJSON-1507438675 tempest-InstanceActionsV221TestJSON-1507438675-project] Getting quotas for project 5bc9ccdaedd3479c84513dff2c7445bc. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 18 22:51:55.192712 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-56e48629-2ffc-4fe5-925f-4ac99458f3f4 tempest-InstanceActionsV221TestJSON-1507438675 tempest-InstanceActionsV221TestJSON-1507438675-project] Getting quotas for user 9799b5e9014b457e9da80c433315d8e0 and project 5bc9ccdaedd3479c84513dff2c7445bc. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 18 22:51:55.204251 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-56e48629-2ffc-4fe5-925f-4ac99458f3f4 tempest-InstanceActionsV221TestJSON-1507438675 tempest-InstanceActionsV221TestJSON-1507438675-project] [instance: 05a7441e-38c5-4a82-ace8-1a1cdd4705b0] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120657) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 18 22:51:55.205027 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-56e48629-2ffc-4fe5-925f-4ac99458f3f4 tempest-InstanceActionsV221TestJSON-1507438675 tempest-InstanceActionsV221TestJSON-1507438675-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:51:55.205456 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-56e48629-2ffc-4fe5-925f-4ac99458f3f4 tempest-InstanceActionsV221TestJSON-1507438675 tempest-InstanceActionsV221TestJSON-1507438675-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:51:55.205839 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-56e48629-2ffc-4fe5-925f-4ac99458f3f4 tempest-InstanceActionsV221TestJSON-1507438675 tempest-InstanceActionsV221TestJSON-1507438675-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:51:55.268083 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-56e48629-2ffc-4fe5-925f-4ac99458f3f4 tempest-InstanceActionsV221TestJSON-1507438675 tempest-InstanceActionsV221TestJSON-1507438675-project] [instance: 05a7441e-38c5-4a82-ace8-1a1cdd4705b0] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='87b0fec9-a948-41f9-a3c0-f0a6a5a8f1e7',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120657) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 18 22:51:55.269306 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-56e48629-2ffc-4fe5-925f-4ac99458f3f4 tempest-InstanceActionsV221TestJSON-1507438675 tempest-InstanceActionsV221TestJSON-1507438675-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:51:55.269738 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-56e48629-2ffc-4fe5-925f-4ac99458f3f4 tempest-InstanceActionsV221TestJSON-1507438675 tempest-InstanceActionsV221TestJSON-1507438675-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:51:55.270193 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-56e48629-2ffc-4fe5-925f-4ac99458f3f4 tempest-InstanceActionsV221TestJSON-1507438675 tempest-InstanceActionsV221TestJSON-1507438675-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:51:55.502016 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-56e48629-2ffc-4fe5-925f-4ac99458f3f4 tempest-InstanceActionsV221TestJSON-1507438675 tempest-InstanceActionsV221TestJSON-1507438675-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:51:55.502459 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-56e48629-2ffc-4fe5-925f-4ac99458f3f4 tempest-InstanceActionsV221TestJSON-1507438675 tempest-InstanceActionsV221TestJSON-1507438675-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:51:55.504368 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-56e48629-2ffc-4fe5-925f-4ac99458f3f4 tempest-InstanceActionsV221TestJSON-1507438675 tempest-InstanceActionsV221TestJSON-1507438675-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:51:55.504368 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-56e48629-2ffc-4fe5-925f-4ac99458f3f4 tempest-InstanceActionsV221TestJSON-1507438675 tempest-InstanceActionsV221TestJSON-1507438675-project] Acquiring lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 18 22:51:55.504368 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-56e48629-2ffc-4fe5-925f-4ac99458f3f4 tempest-InstanceActionsV221TestJSON-1507438675 tempest-InstanceActionsV221TestJSON-1507438675-project] Acquired lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 18 22:51:55.504687 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-3484e196-089d-40df-abaf-d58898015239 None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:51:55.505107 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-3484e196-089d-40df-abaf-d58898015239 None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:51:55.505482 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-3484e196-089d-40df-abaf-d58898015239 None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:51:55.506129 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-3484e196-089d-40df-abaf-d58898015239 None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:51:55.506533 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-3484e196-089d-40df-abaf-d58898015239 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:51:55.506937 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-3484e196-089d-40df-abaf-d58898015239 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:51:55.515492 n-d-853704-1 nova-conductor[120657]: ERROR nova.compute.rpcapi [None req-3484e196-089d-40df-abaf-d58898015239 None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 18 22:51:55.516467 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-3484e196-089d-40df-abaf-d58898015239 None None] Releasing lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 18 22:51:55.517381 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server [None req-3484e196-089d-40df-abaf-d58898015239 None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:51:55.517381 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:51:55.517381 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 18 22:51:55.517381 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 18 22:51:55.517381 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 18 22:51:55.517381 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 22:51:55.517381 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 18 22:51:55.517381 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 22:51:55.517381 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:51:55.517381 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 18 22:51:55.517381 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 18 22:51:55.517381 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 18 22:51:55.517381 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 18 22:51:55.517381 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 18 22:51:55.517381 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 18 22:51:55.517381 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 18 22:51:55.517381 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 18 22:51:55.517381 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 18 22:51:55.517381 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 18 22:51:55.517381 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 18 22:51:55.517381 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 18 22:51:55.517381 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 18 22:51:55.517381 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 18 22:51:55.517381 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:51:55.517381 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 22:52:05.336265 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=14,nova_api:UPDATE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:52:05.420168 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=10,nova_api:DELETE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:52:05.512533 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:52:05.514067 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell1:SELECT=11,nova_cell1:SAVEPOINT=1,nova_cell1:INSERT=18,nova_cell1:RELEASE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:52:13.229702 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=10,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:52:13.231812 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=4,nova_cell0:UPDATE=2 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:52:38.394273 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-76c07e6d-f223-48bd-87ee-21ac0904ea07 tempest-TestNetworkBasicOps-944734306 tempest-TestNetworkBasicOps-944734306-project] Took 0.34 seconds to select destinations for 1 instance(s). {{(pid=120658) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 18 22:52:38.417430 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-76c07e6d-f223-48bd-87ee-21ac0904ea07 tempest-TestNetworkBasicOps-944734306 tempest-TestNetworkBasicOps-944734306-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:52:38.420477 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-76c07e6d-f223-48bd-87ee-21ac0904ea07 tempest-TestNetworkBasicOps-944734306 tempest-TestNetworkBasicOps-944734306-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:52:38.420477 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-76c07e6d-f223-48bd-87ee-21ac0904ea07 tempest-TestNetworkBasicOps-944734306 tempest-TestNetworkBasicOps-944734306-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:52:38.585519 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-76c07e6d-f223-48bd-87ee-21ac0904ea07 tempest-TestNetworkBasicOps-944734306 tempest-TestNetworkBasicOps-944734306-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:52:38.585992 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-76c07e6d-f223-48bd-87ee-21ac0904ea07 tempest-TestNetworkBasicOps-944734306 tempest-TestNetworkBasicOps-944734306-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:52:38.586364 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-76c07e6d-f223-48bd-87ee-21ac0904ea07 tempest-TestNetworkBasicOps-944734306 tempest-TestNetworkBasicOps-944734306-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:52:38.587021 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-76c07e6d-f223-48bd-87ee-21ac0904ea07 tempest-TestNetworkBasicOps-944734306 tempest-TestNetworkBasicOps-944734306-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:52:38.587430 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-76c07e6d-f223-48bd-87ee-21ac0904ea07 tempest-TestNetworkBasicOps-944734306 tempest-TestNetworkBasicOps-944734306-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:52:38.587838 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-76c07e6d-f223-48bd-87ee-21ac0904ea07 tempest-TestNetworkBasicOps-944734306 tempest-TestNetworkBasicOps-944734306-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:52:38.602974 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-76c07e6d-f223-48bd-87ee-21ac0904ea07 tempest-TestNetworkBasicOps-944734306 tempest-TestNetworkBasicOps-944734306-project] Getting quotas for project f0fb1cb4f0c54eb8906e4f1e8d0bf5d5. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 18 22:52:38.607537 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-76c07e6d-f223-48bd-87ee-21ac0904ea07 tempest-TestNetworkBasicOps-944734306 tempest-TestNetworkBasicOps-944734306-project] Getting quotas for user 6460f85405fd4044bbb8edb65ad11f7a and project f0fb1cb4f0c54eb8906e4f1e8d0bf5d5. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 18 22:52:38.620120 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-76c07e6d-f223-48bd-87ee-21ac0904ea07 tempest-TestNetworkBasicOps-944734306 tempest-TestNetworkBasicOps-944734306-project] [instance: e22b2c02-24c2-49c0-afff-da09a9212666] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120658) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 18 22:52:38.620898 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-76c07e6d-f223-48bd-87ee-21ac0904ea07 tempest-TestNetworkBasicOps-944734306 tempest-TestNetworkBasicOps-944734306-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:52:38.621334 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-76c07e6d-f223-48bd-87ee-21ac0904ea07 tempest-TestNetworkBasicOps-944734306 tempest-TestNetworkBasicOps-944734306-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:52:38.621725 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-76c07e6d-f223-48bd-87ee-21ac0904ea07 tempest-TestNetworkBasicOps-944734306 tempest-TestNetworkBasicOps-944734306-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:52:38.688792 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-76c07e6d-f223-48bd-87ee-21ac0904ea07 tempest-TestNetworkBasicOps-944734306 tempest-TestNetworkBasicOps-944734306-project] [instance: e22b2c02-24c2-49c0-afff-da09a9212666] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='87b0fec9-a948-41f9-a3c0-f0a6a5a8f1e7',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120658) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 18 22:52:38.690353 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-76c07e6d-f223-48bd-87ee-21ac0904ea07 tempest-TestNetworkBasicOps-944734306 tempest-TestNetworkBasicOps-944734306-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:52:38.690787 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-76c07e6d-f223-48bd-87ee-21ac0904ea07 tempest-TestNetworkBasicOps-944734306 tempest-TestNetworkBasicOps-944734306-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:52:38.691173 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-76c07e6d-f223-48bd-87ee-21ac0904ea07 tempest-TestNetworkBasicOps-944734306 tempest-TestNetworkBasicOps-944734306-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:52:38.930148 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-76c07e6d-f223-48bd-87ee-21ac0904ea07 tempest-TestNetworkBasicOps-944734306 tempest-TestNetworkBasicOps-944734306-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:52:38.930584 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-76c07e6d-f223-48bd-87ee-21ac0904ea07 tempest-TestNetworkBasicOps-944734306 tempest-TestNetworkBasicOps-944734306-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:52:38.930972 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-76c07e6d-f223-48bd-87ee-21ac0904ea07 tempest-TestNetworkBasicOps-944734306 tempest-TestNetworkBasicOps-944734306-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:52:38.933521 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-76c07e6d-f223-48bd-87ee-21ac0904ea07 tempest-TestNetworkBasicOps-944734306 tempest-TestNetworkBasicOps-944734306-project] Acquiring lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 18 22:52:38.933521 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-76c07e6d-f223-48bd-87ee-21ac0904ea07 tempest-TestNetworkBasicOps-944734306 tempest-TestNetworkBasicOps-944734306-project] Acquired lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 18 22:52:38.933521 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-851620dd-2462-490b-a248-42c441d20443 None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:52:38.933521 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-851620dd-2462-490b-a248-42c441d20443 None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:52:38.933521 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-851620dd-2462-490b-a248-42c441d20443 None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:52:38.934410 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-851620dd-2462-490b-a248-42c441d20443 None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:52:38.934410 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-851620dd-2462-490b-a248-42c441d20443 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:52:38.934723 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-851620dd-2462-490b-a248-42c441d20443 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:52:38.943659 n-d-853704-1 nova-conductor[120658]: ERROR nova.compute.rpcapi [None req-851620dd-2462-490b-a248-42c441d20443 None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 18 22:52:38.944021 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-851620dd-2462-490b-a248-42c441d20443 None None] Releasing lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 18 22:52:38.944943 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server [None req-851620dd-2462-490b-a248-42c441d20443 None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:52:38.944943 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:52:38.944943 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 18 22:52:38.944943 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 18 22:52:38.944943 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 18 22:52:38.944943 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 22:52:38.944943 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 18 22:52:38.944943 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 22:52:38.944943 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:52:38.944943 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 18 22:52:38.944943 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 18 22:52:38.944943 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 18 22:52:38.944943 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 18 22:52:38.944943 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 18 22:52:38.944943 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 18 22:52:38.944943 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 18 22:52:38.944943 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 18 22:52:38.944943 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 18 22:52:38.944943 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 18 22:52:38.944943 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 18 22:52:38.944943 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 18 22:52:38.944943 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 18 22:52:38.944943 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 18 22:52:38.944943 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:52:38.944943 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 22:52:48.868701 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=15 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:52:48.869682 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=9,nova_api:UPDATE=1,nova_api:DELETE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:52:48.940967 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:52:48.946957 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell1:SELECT=11,nova_cell1:INSERT=17 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:52:53.233887 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:52:53.236474 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=15,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:52:59.237907 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-6b1dd2c0-3670-4540-8468-24cb577bd322 tempest-InstanceActionsNegativeTestJSON-1465928983 tempest-InstanceActionsNegativeTestJSON-1465928983-project] Took 0.64 seconds to select destinations for 1 instance(s). {{(pid=120657) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 18 22:52:59.260113 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-6b1dd2c0-3670-4540-8468-24cb577bd322 tempest-InstanceActionsNegativeTestJSON-1465928983 tempest-InstanceActionsNegativeTestJSON-1465928983-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:52:59.260553 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-6b1dd2c0-3670-4540-8468-24cb577bd322 tempest-InstanceActionsNegativeTestJSON-1465928983 tempest-InstanceActionsNegativeTestJSON-1465928983-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:52:59.260947 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-6b1dd2c0-3670-4540-8468-24cb577bd322 tempest-InstanceActionsNegativeTestJSON-1465928983 tempest-InstanceActionsNegativeTestJSON-1465928983-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:52:59.535168 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-6b1dd2c0-3670-4540-8468-24cb577bd322 tempest-InstanceActionsNegativeTestJSON-1465928983 tempest-InstanceActionsNegativeTestJSON-1465928983-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:52:59.535610 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-6b1dd2c0-3670-4540-8468-24cb577bd322 tempest-InstanceActionsNegativeTestJSON-1465928983 tempest-InstanceActionsNegativeTestJSON-1465928983-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:52:59.536037 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-6b1dd2c0-3670-4540-8468-24cb577bd322 tempest-InstanceActionsNegativeTestJSON-1465928983 tempest-InstanceActionsNegativeTestJSON-1465928983-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:52:59.536693 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-6b1dd2c0-3670-4540-8468-24cb577bd322 tempest-InstanceActionsNegativeTestJSON-1465928983 tempest-InstanceActionsNegativeTestJSON-1465928983-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:52:59.537106 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-6b1dd2c0-3670-4540-8468-24cb577bd322 tempest-InstanceActionsNegativeTestJSON-1465928983 tempest-InstanceActionsNegativeTestJSON-1465928983-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:52:59.537489 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-6b1dd2c0-3670-4540-8468-24cb577bd322 tempest-InstanceActionsNegativeTestJSON-1465928983 tempest-InstanceActionsNegativeTestJSON-1465928983-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:52:59.553300 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-6b1dd2c0-3670-4540-8468-24cb577bd322 tempest-InstanceActionsNegativeTestJSON-1465928983 tempest-InstanceActionsNegativeTestJSON-1465928983-project] Getting quotas for project a2a06556298e4a1ab0ed5c6148735197. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 18 22:52:59.557795 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-6b1dd2c0-3670-4540-8468-24cb577bd322 tempest-InstanceActionsNegativeTestJSON-1465928983 tempest-InstanceActionsNegativeTestJSON-1465928983-project] Getting quotas for user 737a362090844b59b3ff5c6d09ae5e87 and project a2a06556298e4a1ab0ed5c6148735197. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 18 22:52:59.570067 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-6b1dd2c0-3670-4540-8468-24cb577bd322 tempest-InstanceActionsNegativeTestJSON-1465928983 tempest-InstanceActionsNegativeTestJSON-1465928983-project] [instance: ca1ba2be-3843-4895-a58e-4822c15af2c9] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120657) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 18 22:52:59.570846 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-6b1dd2c0-3670-4540-8468-24cb577bd322 tempest-InstanceActionsNegativeTestJSON-1465928983 tempest-InstanceActionsNegativeTestJSON-1465928983-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:52:59.571279 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-6b1dd2c0-3670-4540-8468-24cb577bd322 tempest-InstanceActionsNegativeTestJSON-1465928983 tempest-InstanceActionsNegativeTestJSON-1465928983-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:52:59.571681 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-6b1dd2c0-3670-4540-8468-24cb577bd322 tempest-InstanceActionsNegativeTestJSON-1465928983 tempest-InstanceActionsNegativeTestJSON-1465928983-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:52:59.633391 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-6b1dd2c0-3670-4540-8468-24cb577bd322 tempest-InstanceActionsNegativeTestJSON-1465928983 tempest-InstanceActionsNegativeTestJSON-1465928983-project] [instance: ca1ba2be-3843-4895-a58e-4822c15af2c9] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='87b0fec9-a948-41f9-a3c0-f0a6a5a8f1e7',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120657) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 18 22:52:59.634569 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-6b1dd2c0-3670-4540-8468-24cb577bd322 tempest-InstanceActionsNegativeTestJSON-1465928983 tempest-InstanceActionsNegativeTestJSON-1465928983-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:52:59.635009 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-6b1dd2c0-3670-4540-8468-24cb577bd322 tempest-InstanceActionsNegativeTestJSON-1465928983 tempest-InstanceActionsNegativeTestJSON-1465928983-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:52:59.635396 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-6b1dd2c0-3670-4540-8468-24cb577bd322 tempest-InstanceActionsNegativeTestJSON-1465928983 tempest-InstanceActionsNegativeTestJSON-1465928983-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:52:59.849612 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-6b1dd2c0-3670-4540-8468-24cb577bd322 tempest-InstanceActionsNegativeTestJSON-1465928983 tempest-InstanceActionsNegativeTestJSON-1465928983-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:52:59.850401 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-6b1dd2c0-3670-4540-8468-24cb577bd322 tempest-InstanceActionsNegativeTestJSON-1465928983 tempest-InstanceActionsNegativeTestJSON-1465928983-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:52:59.850873 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-6b1dd2c0-3670-4540-8468-24cb577bd322 tempest-InstanceActionsNegativeTestJSON-1465928983 tempest-InstanceActionsNegativeTestJSON-1465928983-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:52:59.851436 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-6b1dd2c0-3670-4540-8468-24cb577bd322 tempest-InstanceActionsNegativeTestJSON-1465928983 tempest-InstanceActionsNegativeTestJSON-1465928983-project] Acquiring lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 18 22:52:59.851993 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-6b1dd2c0-3670-4540-8468-24cb577bd322 tempest-InstanceActionsNegativeTestJSON-1465928983 tempest-InstanceActionsNegativeTestJSON-1465928983-project] Acquired lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 18 22:52:59.852973 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-813ca2e3-98d8-46a8-a58e-fb3f696310bc None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:52:59.853394 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-813ca2e3-98d8-46a8-a58e-fb3f696310bc None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:52:59.853844 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-813ca2e3-98d8-46a8-a58e-fb3f696310bc None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:52:59.855032 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-813ca2e3-98d8-46a8-a58e-fb3f696310bc None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:52:59.855621 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-813ca2e3-98d8-46a8-a58e-fb3f696310bc None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:52:59.857553 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-813ca2e3-98d8-46a8-a58e-fb3f696310bc None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:52:59.865253 n-d-853704-1 nova-conductor[120657]: ERROR nova.compute.rpcapi [None req-813ca2e3-98d8-46a8-a58e-fb3f696310bc None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 18 22:52:59.865651 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-813ca2e3-98d8-46a8-a58e-fb3f696310bc None None] Releasing lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 18 22:52:59.866863 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server [None req-813ca2e3-98d8-46a8-a58e-fb3f696310bc None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:52:59.866863 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:52:59.866863 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 18 22:52:59.866863 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 18 22:52:59.866863 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 18 22:52:59.866863 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 22:52:59.866863 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 18 22:52:59.866863 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 22:52:59.866863 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:52:59.866863 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 18 22:52:59.866863 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 18 22:52:59.866863 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 18 22:52:59.866863 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 18 22:52:59.866863 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 18 22:52:59.866863 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 18 22:52:59.866863 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 18 22:52:59.866863 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 18 22:52:59.866863 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 18 22:52:59.866863 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 18 22:52:59.866863 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 18 22:52:59.866863 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 18 22:52:59.866863 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 18 22:52:59.866863 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 18 22:52:59.866863 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:52:59.866863 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 22:53:09.726567 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=12 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:53:09.787125 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=12,nova_api:UPDATE=1,nova_api:DELETE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:53:09.863931 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:53:09.867727 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell1:SELECT=11,nova_cell1:SAVEPOINT=1,nova_cell1:INSERT=18,nova_cell1:RELEASE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:53:21.338613 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-09580d34-c922-42d1-ad83-faeec4286813 tempest-ServerGroupTestJSON-1064929753 tempest-ServerGroupTestJSON-1064929753-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:53:21.338613 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-09580d34-c922-42d1-ad83-faeec4286813 tempest-ServerGroupTestJSON-1064929753 tempest-ServerGroupTestJSON-1064929753-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:53:21.338613 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-09580d34-c922-42d1-ad83-faeec4286813 tempest-ServerGroupTestJSON-1064929753 tempest-ServerGroupTestJSON-1064929753-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:53:21.904150 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-09580d34-c922-42d1-ad83-faeec4286813 tempest-ServerGroupTestJSON-1064929753 tempest-ServerGroupTestJSON-1064929753-project] Took 0.52 seconds to select destinations for 1 instance(s). {{(pid=120658) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 18 22:53:21.924948 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-09580d34-c922-42d1-ad83-faeec4286813 tempest-ServerGroupTestJSON-1064929753 tempest-ServerGroupTestJSON-1064929753-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:53:21.925471 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-09580d34-c922-42d1-ad83-faeec4286813 tempest-ServerGroupTestJSON-1064929753 tempest-ServerGroupTestJSON-1064929753-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:53:21.926054 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-09580d34-c922-42d1-ad83-faeec4286813 tempest-ServerGroupTestJSON-1064929753 tempest-ServerGroupTestJSON-1064929753-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:53:22.112380 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-09580d34-c922-42d1-ad83-faeec4286813 tempest-ServerGroupTestJSON-1064929753 tempest-ServerGroupTestJSON-1064929753-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:53:22.112802 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-09580d34-c922-42d1-ad83-faeec4286813 tempest-ServerGroupTestJSON-1064929753 tempest-ServerGroupTestJSON-1064929753-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:53:22.113198 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-09580d34-c922-42d1-ad83-faeec4286813 tempest-ServerGroupTestJSON-1064929753 tempest-ServerGroupTestJSON-1064929753-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:53:22.113862 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-09580d34-c922-42d1-ad83-faeec4286813 tempest-ServerGroupTestJSON-1064929753 tempest-ServerGroupTestJSON-1064929753-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:53:22.114273 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-09580d34-c922-42d1-ad83-faeec4286813 tempest-ServerGroupTestJSON-1064929753 tempest-ServerGroupTestJSON-1064929753-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:53:22.114660 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-09580d34-c922-42d1-ad83-faeec4286813 tempest-ServerGroupTestJSON-1064929753 tempest-ServerGroupTestJSON-1064929753-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:53:22.131526 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-09580d34-c922-42d1-ad83-faeec4286813 tempest-ServerGroupTestJSON-1064929753 tempest-ServerGroupTestJSON-1064929753-project] Getting quotas for project bba276574145486f873fc01c7111c540. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 18 22:53:22.135937 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-09580d34-c922-42d1-ad83-faeec4286813 tempest-ServerGroupTestJSON-1064929753 tempest-ServerGroupTestJSON-1064929753-project] Getting quotas for user 11c99f906b5640f98c448ef987b2dfcb and project bba276574145486f873fc01c7111c540. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 18 22:53:22.146689 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-09580d34-c922-42d1-ad83-faeec4286813 tempest-ServerGroupTestJSON-1064929753 tempest-ServerGroupTestJSON-1064929753-project] [instance: 55d41fa0-bfbc-4695-b163-50c318501bba] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120658) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 18 22:53:22.147516 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-09580d34-c922-42d1-ad83-faeec4286813 tempest-ServerGroupTestJSON-1064929753 tempest-ServerGroupTestJSON-1064929753-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:53:22.147976 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-09580d34-c922-42d1-ad83-faeec4286813 tempest-ServerGroupTestJSON-1064929753 tempest-ServerGroupTestJSON-1064929753-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:53:22.148365 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-09580d34-c922-42d1-ad83-faeec4286813 tempest-ServerGroupTestJSON-1064929753 tempest-ServerGroupTestJSON-1064929753-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:53:22.204589 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-09580d34-c922-42d1-ad83-faeec4286813 tempest-ServerGroupTestJSON-1064929753 tempest-ServerGroupTestJSON-1064929753-project] [instance: 55d41fa0-bfbc-4695-b163-50c318501bba] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='87b0fec9-a948-41f9-a3c0-f0a6a5a8f1e7',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120658) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 18 22:53:22.205792 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-09580d34-c922-42d1-ad83-faeec4286813 tempest-ServerGroupTestJSON-1064929753 tempest-ServerGroupTestJSON-1064929753-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:53:22.206199 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-09580d34-c922-42d1-ad83-faeec4286813 tempest-ServerGroupTestJSON-1064929753 tempest-ServerGroupTestJSON-1064929753-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:53:22.206711 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-09580d34-c922-42d1-ad83-faeec4286813 tempest-ServerGroupTestJSON-1064929753 tempest-ServerGroupTestJSON-1064929753-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:53:22.432073 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-09580d34-c922-42d1-ad83-faeec4286813 tempest-ServerGroupTestJSON-1064929753 tempest-ServerGroupTestJSON-1064929753-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:53:22.432734 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-09580d34-c922-42d1-ad83-faeec4286813 tempest-ServerGroupTestJSON-1064929753 tempest-ServerGroupTestJSON-1064929753-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:53:22.432840 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-09580d34-c922-42d1-ad83-faeec4286813 tempest-ServerGroupTestJSON-1064929753 tempest-ServerGroupTestJSON-1064929753-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:53:22.433274 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-09580d34-c922-42d1-ad83-faeec4286813 tempest-ServerGroupTestJSON-1064929753 tempest-ServerGroupTestJSON-1064929753-project] Acquiring lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 18 22:53:22.433624 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-09580d34-c922-42d1-ad83-faeec4286813 tempest-ServerGroupTestJSON-1064929753 tempest-ServerGroupTestJSON-1064929753-project] Acquired lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 18 22:53:22.434365 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-627432f5-f45e-47e9-9f98-7a46d38a3e01 None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:53:22.434774 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-627432f5-f45e-47e9-9f98-7a46d38a3e01 None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:53:22.435156 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-627432f5-f45e-47e9-9f98-7a46d38a3e01 None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:53:22.435859 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-627432f5-f45e-47e9-9f98-7a46d38a3e01 None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:53:22.436274 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-627432f5-f45e-47e9-9f98-7a46d38a3e01 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:53:22.436654 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-627432f5-f45e-47e9-9f98-7a46d38a3e01 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:53:22.445811 n-d-853704-1 nova-conductor[120658]: ERROR nova.compute.rpcapi [None req-627432f5-f45e-47e9-9f98-7a46d38a3e01 None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 18 22:53:22.446206 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-627432f5-f45e-47e9-9f98-7a46d38a3e01 None None] Releasing lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 18 22:53:22.447114 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server [None req-627432f5-f45e-47e9-9f98-7a46d38a3e01 None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:53:22.447114 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:53:22.447114 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 18 22:53:22.447114 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 18 22:53:22.447114 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 18 22:53:22.447114 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 22:53:22.447114 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 18 22:53:22.447114 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 22:53:22.447114 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:53:22.447114 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 18 22:53:22.447114 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 18 22:53:22.447114 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 18 22:53:22.447114 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 18 22:53:22.447114 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 18 22:53:22.447114 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 18 22:53:22.447114 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 18 22:53:22.447114 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 18 22:53:22.447114 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 18 22:53:22.447114 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 18 22:53:22.447114 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 18 22:53:22.447114 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 18 22:53:22.447114 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 18 22:53:22.447114 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 18 22:53:22.447114 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:53:22.447114 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 22:53:32.361139 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=18 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:53:32.362361 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=7,nova_api:UPDATE=1,nova_api:DELETE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:53:32.443964 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:53:32.444772 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell1:SELECT=11,nova_cell1:SAVEPOINT=1,nova_cell1:INSERT=18,nova_cell1:RELEASE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:53:33.230618 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=16,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:53:33.242352 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=14,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:53:34.821231 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-885ff6b8-03b1-4aa0-9eb3-ae9ff14bae7e tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Took 0.38 seconds to select destinations for 1 instance(s). {{(pid=120657) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 18 22:53:34.842166 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-885ff6b8-03b1-4aa0-9eb3-ae9ff14bae7e tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:53:34.842654 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-885ff6b8-03b1-4aa0-9eb3-ae9ff14bae7e tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:53:34.843052 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-885ff6b8-03b1-4aa0-9eb3-ae9ff14bae7e tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:53:34.964035 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-885ff6b8-03b1-4aa0-9eb3-ae9ff14bae7e tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:53:34.964473 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-885ff6b8-03b1-4aa0-9eb3-ae9ff14bae7e tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:53:34.964865 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-885ff6b8-03b1-4aa0-9eb3-ae9ff14bae7e tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:53:34.965532 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-885ff6b8-03b1-4aa0-9eb3-ae9ff14bae7e tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:53:34.965979 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-885ff6b8-03b1-4aa0-9eb3-ae9ff14bae7e tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:53:34.966377 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-885ff6b8-03b1-4aa0-9eb3-ae9ff14bae7e tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:53:34.983466 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-885ff6b8-03b1-4aa0-9eb3-ae9ff14bae7e tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Getting quotas for project 05baaf842a13444d8905683625e5853a. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 18 22:53:34.988101 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-885ff6b8-03b1-4aa0-9eb3-ae9ff14bae7e tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Getting quotas for user 4d73c623dd454555ac8d974cf7a60398 and project 05baaf842a13444d8905683625e5853a. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 18 22:53:34.999202 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-885ff6b8-03b1-4aa0-9eb3-ae9ff14bae7e tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] [instance: f1fae7c1-f8a5-4b9c-8aa9-4722127316e4] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120657) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 18 22:53:35.000004 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-885ff6b8-03b1-4aa0-9eb3-ae9ff14bae7e tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:53:35.000940 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-885ff6b8-03b1-4aa0-9eb3-ae9ff14bae7e tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:53:35.001329 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-885ff6b8-03b1-4aa0-9eb3-ae9ff14bae7e tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:53:35.089592 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-885ff6b8-03b1-4aa0-9eb3-ae9ff14bae7e tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] [instance: f1fae7c1-f8a5-4b9c-8aa9-4722127316e4] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='87b0fec9-a948-41f9-a3c0-f0a6a5a8f1e7',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120657) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 18 22:53:35.090849 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-885ff6b8-03b1-4aa0-9eb3-ae9ff14bae7e tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:53:35.091318 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-885ff6b8-03b1-4aa0-9eb3-ae9ff14bae7e tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:53:35.091820 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-885ff6b8-03b1-4aa0-9eb3-ae9ff14bae7e tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:53:35.271542 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-885ff6b8-03b1-4aa0-9eb3-ae9ff14bae7e tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:53:35.272008 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-885ff6b8-03b1-4aa0-9eb3-ae9ff14bae7e tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:53:35.272433 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-885ff6b8-03b1-4aa0-9eb3-ae9ff14bae7e tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:53:35.482423 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-885ff6b8-03b1-4aa0-9eb3-ae9ff14bae7e tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:53:35.482873 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-885ff6b8-03b1-4aa0-9eb3-ae9ff14bae7e tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:53:35.483270 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-885ff6b8-03b1-4aa0-9eb3-ae9ff14bae7e tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:53:37.090632 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-0e7a5017-14d0-43c7-88ea-8a4a5060f8f8 tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Took 0.39 seconds to select destinations for 1 instance(s). {{(pid=120658) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 18 22:53:37.111318 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-0e7a5017-14d0-43c7-88ea-8a4a5060f8f8 tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:53:37.111793 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-0e7a5017-14d0-43c7-88ea-8a4a5060f8f8 tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:53:37.112872 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-0e7a5017-14d0-43c7-88ea-8a4a5060f8f8 tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:53:37.228566 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-0e7a5017-14d0-43c7-88ea-8a4a5060f8f8 tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:53:37.229008 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-0e7a5017-14d0-43c7-88ea-8a4a5060f8f8 tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:53:37.229397 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-0e7a5017-14d0-43c7-88ea-8a4a5060f8f8 tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:53:37.230093 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-0e7a5017-14d0-43c7-88ea-8a4a5060f8f8 tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:53:37.230500 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-0e7a5017-14d0-43c7-88ea-8a4a5060f8f8 tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:53:37.230880 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-0e7a5017-14d0-43c7-88ea-8a4a5060f8f8 tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:53:37.247346 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-0e7a5017-14d0-43c7-88ea-8a4a5060f8f8 tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Getting quotas for project 05baaf842a13444d8905683625e5853a. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 18 22:53:37.252596 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-0e7a5017-14d0-43c7-88ea-8a4a5060f8f8 tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Getting quotas for user 4d73c623dd454555ac8d974cf7a60398 and project 05baaf842a13444d8905683625e5853a. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 18 22:53:37.263323 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-0e7a5017-14d0-43c7-88ea-8a4a5060f8f8 tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] [instance: 84e33fbe-da08-4557-b1a6-ab19a84956da] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120658) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 18 22:53:37.264129 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-0e7a5017-14d0-43c7-88ea-8a4a5060f8f8 tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:53:37.264568 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-0e7a5017-14d0-43c7-88ea-8a4a5060f8f8 tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:53:37.264960 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-0e7a5017-14d0-43c7-88ea-8a4a5060f8f8 tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:53:37.357667 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-0e7a5017-14d0-43c7-88ea-8a4a5060f8f8 tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] [instance: 84e33fbe-da08-4557-b1a6-ab19a84956da] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='87b0fec9-a948-41f9-a3c0-f0a6a5a8f1e7',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120658) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 18 22:53:37.359322 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-0e7a5017-14d0-43c7-88ea-8a4a5060f8f8 tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:53:37.360048 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-0e7a5017-14d0-43c7-88ea-8a4a5060f8f8 tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:53:37.360606 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-0e7a5017-14d0-43c7-88ea-8a4a5060f8f8 tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:53:37.593444 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-0e7a5017-14d0-43c7-88ea-8a4a5060f8f8 tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:53:37.593880 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-0e7a5017-14d0-43c7-88ea-8a4a5060f8f8 tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:53:37.594400 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-0e7a5017-14d0-43c7-88ea-8a4a5060f8f8 tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:53:37.595005 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-0e7a5017-14d0-43c7-88ea-8a4a5060f8f8 tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Acquiring lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 18 22:53:37.595352 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-0e7a5017-14d0-43c7-88ea-8a4a5060f8f8 tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Acquired lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 18 22:53:37.596157 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-64b862f0-6a5e-4c7d-9c20-7b84fa2381af None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:53:37.596558 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-64b862f0-6a5e-4c7d-9c20-7b84fa2381af None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:53:37.596936 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-64b862f0-6a5e-4c7d-9c20-7b84fa2381af None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:53:37.597567 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-64b862f0-6a5e-4c7d-9c20-7b84fa2381af None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:53:37.597964 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-64b862f0-6a5e-4c7d-9c20-7b84fa2381af None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:53:37.598333 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-64b862f0-6a5e-4c7d-9c20-7b84fa2381af None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:53:37.606625 n-d-853704-1 nova-conductor[120658]: ERROR nova.compute.rpcapi [None req-64b862f0-6a5e-4c7d-9c20-7b84fa2381af None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 18 22:53:37.608753 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-64b862f0-6a5e-4c7d-9c20-7b84fa2381af None None] Releasing lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 18 22:53:37.608753 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server [None req-64b862f0-6a5e-4c7d-9c20-7b84fa2381af None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:53:37.608753 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:53:37.608753 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 18 22:53:37.608753 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 18 22:53:37.608753 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 18 22:53:37.608753 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 22:53:37.608753 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 18 22:53:37.608753 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 22:53:37.608753 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:53:37.608753 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 18 22:53:37.608753 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 18 22:53:37.608753 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 18 22:53:37.608753 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 18 22:53:37.608753 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 18 22:53:37.608753 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 18 22:53:37.608753 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 18 22:53:37.608753 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 18 22:53:37.608753 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 18 22:53:37.608753 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 18 22:53:37.608753 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 18 22:53:37.608753 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 18 22:53:37.608753 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 18 22:53:37.608753 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 18 22:53:37.608753 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:53:37.608753 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 22:53:44.973073 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:53:45.269764 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=15,nova_api:UPDATE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:53:45.270451 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=9,nova_api:DELETE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:53:45.491293 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell1:SELECT=14,nova_cell1:INSERT=17,nova_cell1:UPDATE=12,nova_cell1:DELETE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:53:47.449828 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=14,nova_api:UPDATE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:53:47.521874 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=10,nova_api:DELETE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:53:47.604247 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:53:47.604912 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell1:SELECT=11,nova_cell1:INSERT=17 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:54:13.241102 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=13,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:54:13.242099 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=13,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:54:53.237451 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:54:53.247015 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:55:33.227883 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:55:33.238108 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:55:35.672695 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-9a1720fa-4df3-4ec8-8aa9-2656dd01a39f tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Took 0.46 seconds to select destinations for 1 instance(s). {{(pid=120657) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 18 22:55:35.691931 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-9a1720fa-4df3-4ec8-8aa9-2656dd01a39f tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:55:35.694265 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-9a1720fa-4df3-4ec8-8aa9-2656dd01a39f tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:55:35.694265 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-9a1720fa-4df3-4ec8-8aa9-2656dd01a39f tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:55:35.784613 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-9a1720fa-4df3-4ec8-8aa9-2656dd01a39f tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:55:35.785050 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-9a1720fa-4df3-4ec8-8aa9-2656dd01a39f tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:55:35.785436 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-9a1720fa-4df3-4ec8-8aa9-2656dd01a39f tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:55:35.786087 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-9a1720fa-4df3-4ec8-8aa9-2656dd01a39f tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:55:35.786491 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-9a1720fa-4df3-4ec8-8aa9-2656dd01a39f tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:55:35.786866 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-9a1720fa-4df3-4ec8-8aa9-2656dd01a39f tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:55:35.804601 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-9a1720fa-4df3-4ec8-8aa9-2656dd01a39f tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Getting quotas for project 59dde594f2d04d40886a969d353b0bc5. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 18 22:55:35.809922 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-9a1720fa-4df3-4ec8-8aa9-2656dd01a39f tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Getting quotas for user 372a102cdb374e8e8d8f0ebe31c78e58 and project 59dde594f2d04d40886a969d353b0bc5. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 18 22:55:35.824828 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-9a1720fa-4df3-4ec8-8aa9-2656dd01a39f tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] [instance: 1856d7df-c0ca-4837-8aaa-74844faeb96f] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120657) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 18 22:55:35.824828 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-9a1720fa-4df3-4ec8-8aa9-2656dd01a39f tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:55:35.824828 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-9a1720fa-4df3-4ec8-8aa9-2656dd01a39f tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:55:35.824828 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-9a1720fa-4df3-4ec8-8aa9-2656dd01a39f tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:55:35.871578 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-9a1720fa-4df3-4ec8-8aa9-2656dd01a39f tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] [instance: 1856d7df-c0ca-4837-8aaa-74844faeb96f] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='87b0fec9-a948-41f9-a3c0-f0a6a5a8f1e7',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120657) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 18 22:55:35.872761 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-9a1720fa-4df3-4ec8-8aa9-2656dd01a39f tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:55:35.873209 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-9a1720fa-4df3-4ec8-8aa9-2656dd01a39f tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:55:35.873603 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-9a1720fa-4df3-4ec8-8aa9-2656dd01a39f tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:55:36.042869 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-9a1720fa-4df3-4ec8-8aa9-2656dd01a39f tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:55:36.043306 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-9a1720fa-4df3-4ec8-8aa9-2656dd01a39f tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:55:36.043718 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-9a1720fa-4df3-4ec8-8aa9-2656dd01a39f tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:55:36.044203 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-9a1720fa-4df3-4ec8-8aa9-2656dd01a39f tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Acquiring lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 18 22:55:36.044541 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-9a1720fa-4df3-4ec8-8aa9-2656dd01a39f tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Acquired lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 18 22:55:36.045264 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-dec352a2-5ffc-48ab-8e78-a95604b4b94d None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:55:36.045669 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-dec352a2-5ffc-48ab-8e78-a95604b4b94d None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:55:36.046045 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-dec352a2-5ffc-48ab-8e78-a95604b4b94d None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:55:36.046681 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-dec352a2-5ffc-48ab-8e78-a95604b4b94d None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:55:36.047080 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-dec352a2-5ffc-48ab-8e78-a95604b4b94d None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:55:36.047458 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-dec352a2-5ffc-48ab-8e78-a95604b4b94d None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:55:36.055905 n-d-853704-1 nova-conductor[120657]: ERROR nova.compute.rpcapi [None req-dec352a2-5ffc-48ab-8e78-a95604b4b94d None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 18 22:55:36.056286 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-dec352a2-5ffc-48ab-8e78-a95604b4b94d None None] Releasing lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 18 22:55:36.057900 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server [None req-dec352a2-5ffc-48ab-8e78-a95604b4b94d None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:55:36.057900 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:55:36.057900 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 18 22:55:36.057900 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 18 22:55:36.057900 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 18 22:55:36.057900 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 22:55:36.057900 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 18 22:55:36.057900 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 22:55:36.057900 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:55:36.057900 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 18 22:55:36.057900 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 18 22:55:36.057900 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 18 22:55:36.057900 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 18 22:55:36.057900 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 18 22:55:36.057900 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 18 22:55:36.057900 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 18 22:55:36.057900 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 18 22:55:36.057900 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 18 22:55:36.057900 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 18 22:55:36.057900 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 18 22:55:36.057900 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 18 22:55:36.057900 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 18 22:55:36.057900 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 18 22:55:36.057900 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:55:36.057900 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 22:55:45.931898 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=15 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:55:45.981393 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=9,nova_api:UPDATE=1,nova_api:DELETE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:55:46.052857 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:55:46.054541 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell1:SELECT=11,nova_cell1:INSERT=17 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:56:03.224385 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=11,nova_cell0:UPDATE=3 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:56:03.236871 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=5,nova_cell0:UPDATE=3 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:56:41.287313 n-d-853704-1 nova-conductor[120658]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 18 22:56:43.237159 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:56:43.239737 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:56:55.288572 n-d-853704-1 nova-conductor[120657]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 18 22:57:23.230889 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=7,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:57:23.235099 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:58:03.238940 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:58:03.244124 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:58:04.677329 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-ad498ee7-cee2-4832-a970-d452e80461cc tempest-ServersAaction247Test-1856430120 tempest-ServersAaction247Test-1856430120-project] Took 0.75 seconds to select destinations for 1 instance(s). {{(pid=120658) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 18 22:58:04.701281 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ad498ee7-cee2-4832-a970-d452e80461cc tempest-ServersAaction247Test-1856430120 tempest-ServersAaction247Test-1856430120-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:58:04.701281 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ad498ee7-cee2-4832-a970-d452e80461cc tempest-ServersAaction247Test-1856430120 tempest-ServersAaction247Test-1856430120-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:58:04.701281 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ad498ee7-cee2-4832-a970-d452e80461cc tempest-ServersAaction247Test-1856430120 tempest-ServersAaction247Test-1856430120-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:58:04.935781 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ad498ee7-cee2-4832-a970-d452e80461cc tempest-ServersAaction247Test-1856430120 tempest-ServersAaction247Test-1856430120-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:58:04.936217 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ad498ee7-cee2-4832-a970-d452e80461cc tempest-ServersAaction247Test-1856430120 tempest-ServersAaction247Test-1856430120-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:58:04.936609 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ad498ee7-cee2-4832-a970-d452e80461cc tempest-ServersAaction247Test-1856430120 tempest-ServersAaction247Test-1856430120-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:58:04.937260 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ad498ee7-cee2-4832-a970-d452e80461cc tempest-ServersAaction247Test-1856430120 tempest-ServersAaction247Test-1856430120-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:58:04.937672 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ad498ee7-cee2-4832-a970-d452e80461cc tempest-ServersAaction247Test-1856430120 tempest-ServersAaction247Test-1856430120-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:58:04.938051 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ad498ee7-cee2-4832-a970-d452e80461cc tempest-ServersAaction247Test-1856430120 tempest-ServersAaction247Test-1856430120-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:58:04.953851 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-ad498ee7-cee2-4832-a970-d452e80461cc tempest-ServersAaction247Test-1856430120 tempest-ServersAaction247Test-1856430120-project] Getting quotas for project d72423f049fa4551a0aaacb917a76ad7. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 18 22:58:04.959535 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-ad498ee7-cee2-4832-a970-d452e80461cc tempest-ServersAaction247Test-1856430120 tempest-ServersAaction247Test-1856430120-project] Getting quotas for user 7b7b4213f8db4615bf061a0240dd476a and project d72423f049fa4551a0aaacb917a76ad7. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 18 22:58:04.971439 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-ad498ee7-cee2-4832-a970-d452e80461cc tempest-ServersAaction247Test-1856430120 tempest-ServersAaction247Test-1856430120-project] [instance: cae238be-7068-488c-8de2-f6998a7c3100] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120658) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 18 22:58:04.972497 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ad498ee7-cee2-4832-a970-d452e80461cc tempest-ServersAaction247Test-1856430120 tempest-ServersAaction247Test-1856430120-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:58:04.972783 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ad498ee7-cee2-4832-a970-d452e80461cc tempest-ServersAaction247Test-1856430120 tempest-ServersAaction247Test-1856430120-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:58:04.973116 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ad498ee7-cee2-4832-a970-d452e80461cc tempest-ServersAaction247Test-1856430120 tempest-ServersAaction247Test-1856430120-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:58:05.028956 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-ad498ee7-cee2-4832-a970-d452e80461cc tempest-ServersAaction247Test-1856430120 tempest-ServersAaction247Test-1856430120-project] [instance: cae238be-7068-488c-8de2-f6998a7c3100] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='87b0fec9-a948-41f9-a3c0-f0a6a5a8f1e7',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120658) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 18 22:58:05.030157 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ad498ee7-cee2-4832-a970-d452e80461cc tempest-ServersAaction247Test-1856430120 tempest-ServersAaction247Test-1856430120-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:58:05.030594 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ad498ee7-cee2-4832-a970-d452e80461cc tempest-ServersAaction247Test-1856430120 tempest-ServersAaction247Test-1856430120-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:58:05.031006 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ad498ee7-cee2-4832-a970-d452e80461cc tempest-ServersAaction247Test-1856430120 tempest-ServersAaction247Test-1856430120-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:58:05.218547 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ad498ee7-cee2-4832-a970-d452e80461cc tempest-ServersAaction247Test-1856430120 tempest-ServersAaction247Test-1856430120-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:58:05.223474 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ad498ee7-cee2-4832-a970-d452e80461cc tempest-ServersAaction247Test-1856430120 tempest-ServersAaction247Test-1856430120-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:58:05.224852 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ad498ee7-cee2-4832-a970-d452e80461cc tempest-ServersAaction247Test-1856430120 tempest-ServersAaction247Test-1856430120-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.005s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:58:05.225331 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ad498ee7-cee2-4832-a970-d452e80461cc tempest-ServersAaction247Test-1856430120 tempest-ServersAaction247Test-1856430120-project] Acquiring lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 18 22:58:05.225681 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ad498ee7-cee2-4832-a970-d452e80461cc tempest-ServersAaction247Test-1856430120 tempest-ServersAaction247Test-1856430120-project] Acquired lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 18 22:58:05.226451 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-15cece13-0123-451c-9c56-b158f833d2a5 None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:58:05.226971 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-15cece13-0123-451c-9c56-b158f833d2a5 None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:58:05.227350 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-15cece13-0123-451c-9c56-b158f833d2a5 None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:58:05.228017 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-15cece13-0123-451c-9c56-b158f833d2a5 None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:58:05.228419 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-15cece13-0123-451c-9c56-b158f833d2a5 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:58:05.228793 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-15cece13-0123-451c-9c56-b158f833d2a5 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:58:05.240628 n-d-853704-1 nova-conductor[120658]: ERROR nova.compute.rpcapi [None req-15cece13-0123-451c-9c56-b158f833d2a5 None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 18 22:58:05.241013 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-15cece13-0123-451c-9c56-b158f833d2a5 None None] Releasing lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 18 22:58:05.242357 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server [None req-15cece13-0123-451c-9c56-b158f833d2a5 None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:58:05.242357 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:58:05.242357 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 18 22:58:05.242357 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 18 22:58:05.242357 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 18 22:58:05.242357 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 22:58:05.242357 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 18 22:58:05.242357 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 22:58:05.242357 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:58:05.242357 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 18 22:58:05.242357 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 18 22:58:05.242357 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 18 22:58:05.242357 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 18 22:58:05.242357 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 18 22:58:05.242357 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 18 22:58:05.242357 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 18 22:58:05.242357 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 18 22:58:05.242357 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 18 22:58:05.242357 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 18 22:58:05.242357 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 18 22:58:05.242357 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 18 22:58:05.242357 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 18 22:58:05.242357 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 18 22:58:05.242357 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:58:05.242357 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 22:58:15.090335 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=9,nova_api:UPDATE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:58:15.135409 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=15,nova_api:DELETE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:58:15.239206 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:58:15.243202 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell1:SELECT=11,nova_cell1:SAVEPOINT=1,nova_cell1:INSERT=18,nova_cell1:RELEASE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:58:43.239209 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=14,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:58:43.241171 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:59:17.141286 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-0cc537d8-c0d1-487c-932e-a0c328c2f313 tempest-TestServerBasicOps-1498623685 tempest-TestServerBasicOps-1498623685-project] Took 0.72 seconds to select destinations for 1 instance(s). {{(pid=120657) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 18 22:59:17.166800 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-0cc537d8-c0d1-487c-932e-a0c328c2f313 tempest-TestServerBasicOps-1498623685 tempest-TestServerBasicOps-1498623685-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:59:17.167372 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-0cc537d8-c0d1-487c-932e-a0c328c2f313 tempest-TestServerBasicOps-1498623685 tempest-TestServerBasicOps-1498623685-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:59:17.167925 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-0cc537d8-c0d1-487c-932e-a0c328c2f313 tempest-TestServerBasicOps-1498623685 tempest-TestServerBasicOps-1498623685-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:59:17.380118 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:59:17.383351 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-0cc537d8-c0d1-487c-932e-a0c328c2f313 tempest-TestServerBasicOps-1498623685 tempest-TestServerBasicOps-1498623685-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:59:17.384011 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-0cc537d8-c0d1-487c-932e-a0c328c2f313 tempest-TestServerBasicOps-1498623685 tempest-TestServerBasicOps-1498623685-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:59:17.384565 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-0cc537d8-c0d1-487c-932e-a0c328c2f313 tempest-TestServerBasicOps-1498623685 tempest-TestServerBasicOps-1498623685-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:59:17.385237 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-0cc537d8-c0d1-487c-932e-a0c328c2f313 tempest-TestServerBasicOps-1498623685 tempest-TestServerBasicOps-1498623685-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:59:17.385653 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-0cc537d8-c0d1-487c-932e-a0c328c2f313 tempest-TestServerBasicOps-1498623685 tempest-TestServerBasicOps-1498623685-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:59:17.386035 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-0cc537d8-c0d1-487c-932e-a0c328c2f313 tempest-TestServerBasicOps-1498623685 tempest-TestServerBasicOps-1498623685-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:59:17.406574 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-0cc537d8-c0d1-487c-932e-a0c328c2f313 tempest-TestServerBasicOps-1498623685 tempest-TestServerBasicOps-1498623685-project] Getting quotas for project 43a1b67a9b8e41f189f8b4598d8e361f. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 18 22:59:17.421192 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-0cc537d8-c0d1-487c-932e-a0c328c2f313 tempest-TestServerBasicOps-1498623685 tempest-TestServerBasicOps-1498623685-project] Getting quotas for user d41c0003a76041a7a0dabf45f030e24a and project 43a1b67a9b8e41f189f8b4598d8e361f. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 18 22:59:17.445106 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-0cc537d8-c0d1-487c-932e-a0c328c2f313 tempest-TestServerBasicOps-1498623685 tempest-TestServerBasicOps-1498623685-project] [instance: 4baf2c98-e78a-43a1-9dc8-a1284b5d6837] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120657) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 18 22:59:17.446331 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-0cc537d8-c0d1-487c-932e-a0c328c2f313 tempest-TestServerBasicOps-1498623685 tempest-TestServerBasicOps-1498623685-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:59:17.446961 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-0cc537d8-c0d1-487c-932e-a0c328c2f313 tempest-TestServerBasicOps-1498623685 tempest-TestServerBasicOps-1498623685-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:59:17.447509 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-0cc537d8-c0d1-487c-932e-a0c328c2f313 tempest-TestServerBasicOps-1498623685 tempest-TestServerBasicOps-1498623685-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:59:17.542253 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-0cc537d8-c0d1-487c-932e-a0c328c2f313 tempest-TestServerBasicOps-1498623685 tempest-TestServerBasicOps-1498623685-project] [instance: 4baf2c98-e78a-43a1-9dc8-a1284b5d6837] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='87b0fec9-a948-41f9-a3c0-f0a6a5a8f1e7',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120657) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 18 22:59:17.543911 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-0cc537d8-c0d1-487c-932e-a0c328c2f313 tempest-TestServerBasicOps-1498623685 tempest-TestServerBasicOps-1498623685-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:59:17.544616 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-0cc537d8-c0d1-487c-932e-a0c328c2f313 tempest-TestServerBasicOps-1498623685 tempest-TestServerBasicOps-1498623685-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:59:17.545237 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-0cc537d8-c0d1-487c-932e-a0c328c2f313 tempest-TestServerBasicOps-1498623685 tempest-TestServerBasicOps-1498623685-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:59:17.834632 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-0cc537d8-c0d1-487c-932e-a0c328c2f313 tempest-TestServerBasicOps-1498623685 tempest-TestServerBasicOps-1498623685-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:59:17.835099 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-0cc537d8-c0d1-487c-932e-a0c328c2f313 tempest-TestServerBasicOps-1498623685 tempest-TestServerBasicOps-1498623685-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:59:17.839016 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-0cc537d8-c0d1-487c-932e-a0c328c2f313 tempest-TestServerBasicOps-1498623685 tempest-TestServerBasicOps-1498623685-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:59:17.839016 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-0cc537d8-c0d1-487c-932e-a0c328c2f313 tempest-TestServerBasicOps-1498623685 tempest-TestServerBasicOps-1498623685-project] Acquiring lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 18 22:59:17.839016 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-0cc537d8-c0d1-487c-932e-a0c328c2f313 tempest-TestServerBasicOps-1498623685 tempest-TestServerBasicOps-1498623685-project] Acquired lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 18 22:59:17.839016 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-70ac7ace-fcba-43f3-baa0-484fa09c53fa None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:59:17.839016 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-70ac7ace-fcba-43f3-baa0-484fa09c53fa None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:59:17.839016 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-70ac7ace-fcba-43f3-baa0-484fa09c53fa None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:59:17.840360 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-70ac7ace-fcba-43f3-baa0-484fa09c53fa None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:59:17.840950 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-70ac7ace-fcba-43f3-baa0-484fa09c53fa None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:59:17.841574 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-70ac7ace-fcba-43f3-baa0-484fa09c53fa None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:59:17.851589 n-d-853704-1 nova-conductor[120657]: ERROR nova.compute.rpcapi [None req-70ac7ace-fcba-43f3-baa0-484fa09c53fa None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 18 22:59:17.852026 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-70ac7ace-fcba-43f3-baa0-484fa09c53fa None None] Releasing lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 18 22:59:17.852946 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server [None req-70ac7ace-fcba-43f3-baa0-484fa09c53fa None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:59:17.852946 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:59:17.852946 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 18 22:59:17.852946 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 18 22:59:17.852946 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 18 22:59:17.852946 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 22:59:17.852946 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 18 22:59:17.852946 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 22:59:17.852946 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:59:17.852946 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 18 22:59:17.852946 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 18 22:59:17.852946 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 18 22:59:17.852946 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 18 22:59:17.852946 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 18 22:59:17.852946 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 18 22:59:17.852946 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 18 22:59:17.852946 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 18 22:59:17.852946 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 18 22:59:17.852946 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 18 22:59:17.852946 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 18 22:59:17.852946 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 18 22:59:17.852946 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 18 22:59:17.852946 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 18 22:59:17.852946 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:59:17.852946 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 22:59:21.198540 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-cfba3f16-0bbf-43d1-8201-eaf904c80a0a tempest-MultipleCreateTestJSON-1319949902 tempest-MultipleCreateTestJSON-1319949902-project] Took 0.94 seconds to select destinations for 2 instance(s). {{(pid=120658) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 18 22:59:21.219595 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-cfba3f16-0bbf-43d1-8201-eaf904c80a0a tempest-MultipleCreateTestJSON-1319949902 tempest-MultipleCreateTestJSON-1319949902-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:59:21.220076 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-cfba3f16-0bbf-43d1-8201-eaf904c80a0a tempest-MultipleCreateTestJSON-1319949902 tempest-MultipleCreateTestJSON-1319949902-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:59:21.220781 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-cfba3f16-0bbf-43d1-8201-eaf904c80a0a tempest-MultipleCreateTestJSON-1319949902 tempest-MultipleCreateTestJSON-1319949902-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:59:21.471311 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-cfba3f16-0bbf-43d1-8201-eaf904c80a0a tempest-MultipleCreateTestJSON-1319949902 tempest-MultipleCreateTestJSON-1319949902-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:59:21.472320 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-cfba3f16-0bbf-43d1-8201-eaf904c80a0a tempest-MultipleCreateTestJSON-1319949902 tempest-MultipleCreateTestJSON-1319949902-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:59:21.473685 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-cfba3f16-0bbf-43d1-8201-eaf904c80a0a tempest-MultipleCreateTestJSON-1319949902 tempest-MultipleCreateTestJSON-1319949902-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:59:21.629186 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:59:21.637248 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-cfba3f16-0bbf-43d1-8201-eaf904c80a0a tempest-MultipleCreateTestJSON-1319949902 tempest-MultipleCreateTestJSON-1319949902-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:59:21.637993 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-cfba3f16-0bbf-43d1-8201-eaf904c80a0a tempest-MultipleCreateTestJSON-1319949902 tempest-MultipleCreateTestJSON-1319949902-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:59:21.638677 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-cfba3f16-0bbf-43d1-8201-eaf904c80a0a tempest-MultipleCreateTestJSON-1319949902 tempest-MultipleCreateTestJSON-1319949902-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:59:21.641370 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-cfba3f16-0bbf-43d1-8201-eaf904c80a0a tempest-MultipleCreateTestJSON-1319949902 tempest-MultipleCreateTestJSON-1319949902-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:59:21.644361 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-cfba3f16-0bbf-43d1-8201-eaf904c80a0a tempest-MultipleCreateTestJSON-1319949902 tempest-MultipleCreateTestJSON-1319949902-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.005s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:59:21.644813 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-cfba3f16-0bbf-43d1-8201-eaf904c80a0a tempest-MultipleCreateTestJSON-1319949902 tempest-MultipleCreateTestJSON-1319949902-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:59:21.686660 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-cfba3f16-0bbf-43d1-8201-eaf904c80a0a tempest-MultipleCreateTestJSON-1319949902 tempest-MultipleCreateTestJSON-1319949902-project] Getting quotas for project 9b1d3d3bc3f0421781c885492c70406d. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 18 22:59:21.695242 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-cfba3f16-0bbf-43d1-8201-eaf904c80a0a tempest-MultipleCreateTestJSON-1319949902 tempest-MultipleCreateTestJSON-1319949902-project] Getting quotas for user 618be23c9d424627a3052447bfe8cf43 and project 9b1d3d3bc3f0421781c885492c70406d. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 18 22:59:21.716759 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-cfba3f16-0bbf-43d1-8201-eaf904c80a0a tempest-MultipleCreateTestJSON-1319949902 tempest-MultipleCreateTestJSON-1319949902-project] [instance: cb51508d-896e-47ec-8d73-e70d440234d4] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [] {{(pid=120658) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 18 22:59:21.718147 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-cfba3f16-0bbf-43d1-8201-eaf904c80a0a tempest-MultipleCreateTestJSON-1319949902 tempest-MultipleCreateTestJSON-1319949902-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:59:21.719214 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-cfba3f16-0bbf-43d1-8201-eaf904c80a0a tempest-MultipleCreateTestJSON-1319949902 tempest-MultipleCreateTestJSON-1319949902-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:59:21.719375 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-cfba3f16-0bbf-43d1-8201-eaf904c80a0a tempest-MultipleCreateTestJSON-1319949902 tempest-MultipleCreateTestJSON-1319949902-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:59:22.056416 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-cfba3f16-0bbf-43d1-8201-eaf904c80a0a tempest-MultipleCreateTestJSON-1319949902 tempest-MultipleCreateTestJSON-1319949902-project] [instance: cb51508d-896e-47ec-8d73-e70d440234d4] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='87b0fec9-a948-41f9-a3c0-f0a6a5a8f1e7',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120658) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 18 22:59:22.062015 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-cfba3f16-0bbf-43d1-8201-eaf904c80a0a tempest-MultipleCreateTestJSON-1319949902 tempest-MultipleCreateTestJSON-1319949902-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:59:22.062015 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-cfba3f16-0bbf-43d1-8201-eaf904c80a0a tempest-MultipleCreateTestJSON-1319949902 tempest-MultipleCreateTestJSON-1319949902-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:59:22.062015 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-cfba3f16-0bbf-43d1-8201-eaf904c80a0a tempest-MultipleCreateTestJSON-1319949902 tempest-MultipleCreateTestJSON-1319949902-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:59:22.343375 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-cfba3f16-0bbf-43d1-8201-eaf904c80a0a tempest-MultipleCreateTestJSON-1319949902 tempest-MultipleCreateTestJSON-1319949902-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:59:22.344081 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-cfba3f16-0bbf-43d1-8201-eaf904c80a0a tempest-MultipleCreateTestJSON-1319949902 tempest-MultipleCreateTestJSON-1319949902-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:59:22.344488 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-cfba3f16-0bbf-43d1-8201-eaf904c80a0a tempest-MultipleCreateTestJSON-1319949902 tempest-MultipleCreateTestJSON-1319949902-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:59:22.345227 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-cfba3f16-0bbf-43d1-8201-eaf904c80a0a tempest-MultipleCreateTestJSON-1319949902 tempest-MultipleCreateTestJSON-1319949902-project] Acquiring lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 18 22:59:22.345642 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-cfba3f16-0bbf-43d1-8201-eaf904c80a0a tempest-MultipleCreateTestJSON-1319949902 tempest-MultipleCreateTestJSON-1319949902-project] Acquired lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 18 22:59:22.346561 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-e0c532d6-03c0-430d-82d1-eef30aec57b0 None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:59:22.347277 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-e0c532d6-03c0-430d-82d1-eef30aec57b0 None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:59:22.347757 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-e0c532d6-03c0-430d-82d1-eef30aec57b0 None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:59:22.348537 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-e0c532d6-03c0-430d-82d1-eef30aec57b0 None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:59:22.349021 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-e0c532d6-03c0-430d-82d1-eef30aec57b0 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:59:22.349466 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-e0c532d6-03c0-430d-82d1-eef30aec57b0 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:59:22.360402 n-d-853704-1 nova-conductor[120658]: ERROR nova.compute.rpcapi [None req-e0c532d6-03c0-430d-82d1-eef30aec57b0 None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 18 22:59:22.360793 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-e0c532d6-03c0-430d-82d1-eef30aec57b0 None None] Releasing lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 18 22:59:22.361951 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server [None req-e0c532d6-03c0-430d-82d1-eef30aec57b0 None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:59:22.361951 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:59:22.361951 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 18 22:59:22.361951 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 18 22:59:22.361951 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 18 22:59:22.361951 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 22:59:22.361951 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 18 22:59:22.361951 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 22:59:22.361951 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:59:22.361951 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 18 22:59:22.361951 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 18 22:59:22.361951 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 18 22:59:22.361951 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 18 22:59:22.361951 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 18 22:59:22.361951 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 18 22:59:22.361951 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 18 22:59:22.361951 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 18 22:59:22.361951 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 18 22:59:22.361951 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 18 22:59:22.361951 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 18 22:59:22.361951 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 18 22:59:22.361951 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 18 22:59:22.361951 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 18 22:59:22.361951 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:59:22.361951 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 22:59:27.728745 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=13 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:59:27.729510 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=11,nova_api:UPDATE=1,nova_api:DELETE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:59:27.851760 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:59:27.852511 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell1:SELECT=11,nova_cell1:SAVEPOINT=1,nova_cell1:INSERT=21,nova_cell1:RELEASE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:59:32.198485 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=14 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:59:32.276412 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=16,nova_api:UPDATE=1,nova_api:DELETE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:59:32.359443 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:59:32.360200 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell1:SELECT=16,nova_cell1:SAVEPOINT=1,nova_cell1:INSERT=33,nova_cell1:RELEASE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:59:41.074780 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-7df669d0-0641-483b-abd6-319e1920a154 tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Took 0.46 seconds to select destinations for 1 instance(s). {{(pid=120657) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 18 22:59:41.104103 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-7df669d0-0641-483b-abd6-319e1920a154 tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:59:41.104782 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-7df669d0-0641-483b-abd6-319e1920a154 tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.003s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:59:41.105489 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-7df669d0-0641-483b-abd6-319e1920a154 tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:59:41.253539 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-7df669d0-0641-483b-abd6-319e1920a154 tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:59:41.254114 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-7df669d0-0641-483b-abd6-319e1920a154 tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:59:41.254685 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-7df669d0-0641-483b-abd6-319e1920a154 tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:59:41.255345 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-7df669d0-0641-483b-abd6-319e1920a154 tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:59:41.255806 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-7df669d0-0641-483b-abd6-319e1920a154 tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:59:41.256652 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-7df669d0-0641-483b-abd6-319e1920a154 tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:59:41.274712 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-7df669d0-0641-483b-abd6-319e1920a154 tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Getting quotas for project 05baaf842a13444d8905683625e5853a. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 18 22:59:41.280352 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-7df669d0-0641-483b-abd6-319e1920a154 tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Getting quotas for user 4d73c623dd454555ac8d974cf7a60398 and project 05baaf842a13444d8905683625e5853a. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 18 22:59:41.293099 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-7df669d0-0641-483b-abd6-319e1920a154 tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] [instance: 6b5b6e5d-10a0-4c01-8e8d-baddb44f6c0c] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120657) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 18 22:59:41.294010 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-7df669d0-0641-483b-abd6-319e1920a154 tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:59:41.294435 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-7df669d0-0641-483b-abd6-319e1920a154 tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:59:41.294833 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-7df669d0-0641-483b-abd6-319e1920a154 tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:59:41.359316 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-7df669d0-0641-483b-abd6-319e1920a154 tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] [instance: 6b5b6e5d-10a0-4c01-8e8d-baddb44f6c0c] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='87b0fec9-a948-41f9-a3c0-f0a6a5a8f1e7',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120657) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 18 22:59:41.360388 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-7df669d0-0641-483b-abd6-319e1920a154 tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:59:41.360849 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-7df669d0-0641-483b-abd6-319e1920a154 tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:59:41.361547 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-7df669d0-0641-483b-abd6-319e1920a154 tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:59:41.752830 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-7df669d0-0641-483b-abd6-319e1920a154 tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:59:41.753699 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-7df669d0-0641-483b-abd6-319e1920a154 tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.003s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:59:41.754508 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-7df669d0-0641-483b-abd6-319e1920a154 tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:59:41.755422 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-7df669d0-0641-483b-abd6-319e1920a154 tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Acquiring lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 18 22:59:41.756017 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-7df669d0-0641-483b-abd6-319e1920a154 tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Acquired lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 18 22:59:41.757858 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-f875ab64-3e84-4fca-8192-32360aaf0bc6 None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:59:41.758678 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-f875ab64-3e84-4fca-8192-32360aaf0bc6 None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:59:41.759271 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-f875ab64-3e84-4fca-8192-32360aaf0bc6 None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:59:41.760188 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-f875ab64-3e84-4fca-8192-32360aaf0bc6 None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:59:41.760974 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-f875ab64-3e84-4fca-8192-32360aaf0bc6 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:59:41.761931 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-f875ab64-3e84-4fca-8192-32360aaf0bc6 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:59:41.779471 n-d-853704-1 nova-conductor[120657]: ERROR nova.compute.rpcapi [None req-f875ab64-3e84-4fca-8192-32360aaf0bc6 None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 18 22:59:41.780012 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-f875ab64-3e84-4fca-8192-32360aaf0bc6 None None] Releasing lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 18 22:59:41.781083 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server [None req-f875ab64-3e84-4fca-8192-32360aaf0bc6 None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:59:41.781083 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:59:41.781083 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 18 22:59:41.781083 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 18 22:59:41.781083 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 18 22:59:41.781083 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 22:59:41.781083 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 18 22:59:41.781083 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 22:59:41.781083 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:59:41.781083 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 18 22:59:41.781083 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 18 22:59:41.781083 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 18 22:59:41.781083 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 18 22:59:41.781083 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 18 22:59:41.781083 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 18 22:59:41.781083 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 18 22:59:41.781083 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 18 22:59:41.781083 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 18 22:59:41.781083 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 18 22:59:41.781083 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 18 22:59:41.781083 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 18 22:59:41.781083 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 18 22:59:41.781083 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 18 22:59:41.781083 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:59:41.781083 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 22:59:49.050904 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-d68c1891-985f-4198-bad4-0855a108ec57 tempest-VolumesSnapshotTestJSON-1490689032 tempest-VolumesSnapshotTestJSON-1490689032-project] Took 0.60 seconds to select destinations for 1 instance(s). {{(pid=120658) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 18 22:59:49.071601 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-d68c1891-985f-4198-bad4-0855a108ec57 tempest-VolumesSnapshotTestJSON-1490689032 tempest-VolumesSnapshotTestJSON-1490689032-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:59:49.072108 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-d68c1891-985f-4198-bad4-0855a108ec57 tempest-VolumesSnapshotTestJSON-1490689032 tempest-VolumesSnapshotTestJSON-1490689032-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:59:49.072443 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-d68c1891-985f-4198-bad4-0855a108ec57 tempest-VolumesSnapshotTestJSON-1490689032 tempest-VolumesSnapshotTestJSON-1490689032-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:59:49.199400 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-d68c1891-985f-4198-bad4-0855a108ec57 tempest-VolumesSnapshotTestJSON-1490689032 tempest-VolumesSnapshotTestJSON-1490689032-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:59:49.200107 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-d68c1891-985f-4198-bad4-0855a108ec57 tempest-VolumesSnapshotTestJSON-1490689032 tempest-VolumesSnapshotTestJSON-1490689032-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:59:49.200469 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-d68c1891-985f-4198-bad4-0855a108ec57 tempest-VolumesSnapshotTestJSON-1490689032 tempest-VolumesSnapshotTestJSON-1490689032-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:59:49.201199 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-d68c1891-985f-4198-bad4-0855a108ec57 tempest-VolumesSnapshotTestJSON-1490689032 tempest-VolumesSnapshotTestJSON-1490689032-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:59:49.201618 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-d68c1891-985f-4198-bad4-0855a108ec57 tempest-VolumesSnapshotTestJSON-1490689032 tempest-VolumesSnapshotTestJSON-1490689032-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:59:49.202002 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-d68c1891-985f-4198-bad4-0855a108ec57 tempest-VolumesSnapshotTestJSON-1490689032 tempest-VolumesSnapshotTestJSON-1490689032-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:59:49.218281 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-d68c1891-985f-4198-bad4-0855a108ec57 tempest-VolumesSnapshotTestJSON-1490689032 tempest-VolumesSnapshotTestJSON-1490689032-project] Getting quotas for project 6ab620df4d7d41738193bb1eb0ad93a1. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 18 22:59:49.222724 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-d68c1891-985f-4198-bad4-0855a108ec57 tempest-VolumesSnapshotTestJSON-1490689032 tempest-VolumesSnapshotTestJSON-1490689032-project] Getting quotas for user 333a292527ac4188aaa4a20f192eb9a6 and project 6ab620df4d7d41738193bb1eb0ad93a1. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 18 22:59:49.233677 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-d68c1891-985f-4198-bad4-0855a108ec57 tempest-VolumesSnapshotTestJSON-1490689032 tempest-VolumesSnapshotTestJSON-1490689032-project] [instance: 9f7540bc-36cf-4f5b-9d1a-222aee186eb2] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120658) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 18 22:59:49.234500 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-d68c1891-985f-4198-bad4-0855a108ec57 tempest-VolumesSnapshotTestJSON-1490689032 tempest-VolumesSnapshotTestJSON-1490689032-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:59:49.234927 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-d68c1891-985f-4198-bad4-0855a108ec57 tempest-VolumesSnapshotTestJSON-1490689032 tempest-VolumesSnapshotTestJSON-1490689032-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:59:49.235317 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-d68c1891-985f-4198-bad4-0855a108ec57 tempest-VolumesSnapshotTestJSON-1490689032 tempest-VolumesSnapshotTestJSON-1490689032-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:59:49.295828 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-d68c1891-985f-4198-bad4-0855a108ec57 tempest-VolumesSnapshotTestJSON-1490689032 tempest-VolumesSnapshotTestJSON-1490689032-project] [instance: 9f7540bc-36cf-4f5b-9d1a-222aee186eb2] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='87b0fec9-a948-41f9-a3c0-f0a6a5a8f1e7',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120658) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 18 22:59:49.297237 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-d68c1891-985f-4198-bad4-0855a108ec57 tempest-VolumesSnapshotTestJSON-1490689032 tempest-VolumesSnapshotTestJSON-1490689032-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:59:49.297671 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-d68c1891-985f-4198-bad4-0855a108ec57 tempest-VolumesSnapshotTestJSON-1490689032 tempest-VolumesSnapshotTestJSON-1490689032-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:59:49.298082 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-d68c1891-985f-4198-bad4-0855a108ec57 tempest-VolumesSnapshotTestJSON-1490689032 tempest-VolumesSnapshotTestJSON-1490689032-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:59:49.525045 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-d68c1891-985f-4198-bad4-0855a108ec57 tempest-VolumesSnapshotTestJSON-1490689032 tempest-VolumesSnapshotTestJSON-1490689032-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:59:49.525482 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-d68c1891-985f-4198-bad4-0855a108ec57 tempest-VolumesSnapshotTestJSON-1490689032 tempest-VolumesSnapshotTestJSON-1490689032-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:59:49.525892 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-d68c1891-985f-4198-bad4-0855a108ec57 tempest-VolumesSnapshotTestJSON-1490689032 tempest-VolumesSnapshotTestJSON-1490689032-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:59:49.526333 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-d68c1891-985f-4198-bad4-0855a108ec57 tempest-VolumesSnapshotTestJSON-1490689032 tempest-VolumesSnapshotTestJSON-1490689032-project] Acquiring lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 18 22:59:49.526678 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-d68c1891-985f-4198-bad4-0855a108ec57 tempest-VolumesSnapshotTestJSON-1490689032 tempest-VolumesSnapshotTestJSON-1490689032-project] Acquired lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 18 22:59:49.527420 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-4ff57d62-7beb-4d83-8ede-d2bc6fb8eade None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:59:49.527856 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-4ff57d62-7beb-4d83-8ede-d2bc6fb8eade None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:59:49.528229 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-4ff57d62-7beb-4d83-8ede-d2bc6fb8eade None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:59:49.528863 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-4ff57d62-7beb-4d83-8ede-d2bc6fb8eade None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 22:59:49.529260 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-4ff57d62-7beb-4d83-8ede-d2bc6fb8eade None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 22:59:49.529629 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-4ff57d62-7beb-4d83-8ede-d2bc6fb8eade None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 22:59:49.539238 n-d-853704-1 nova-conductor[120658]: ERROR nova.compute.rpcapi [None req-4ff57d62-7beb-4d83-8ede-d2bc6fb8eade None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 18 22:59:49.539620 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-4ff57d62-7beb-4d83-8ede-d2bc6fb8eade None None] Releasing lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 18 22:59:49.540542 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server [None req-4ff57d62-7beb-4d83-8ede-d2bc6fb8eade None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:59:49.540542 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:59:49.540542 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 18 22:59:49.540542 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 18 22:59:49.540542 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 18 22:59:49.540542 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 22:59:49.540542 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 18 22:59:49.540542 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 22:59:49.540542 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 22:59:49.540542 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 18 22:59:49.540542 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 18 22:59:49.540542 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 18 22:59:49.540542 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 18 22:59:49.540542 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 18 22:59:49.540542 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 18 22:59:49.540542 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 18 22:59:49.540542 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 18 22:59:49.540542 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 18 22:59:49.540542 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 18 22:59:49.540542 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 18 22:59:49.540542 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 18 22:59:49.540542 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 18 22:59:49.540542 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 18 22:59:49.540542 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 22:59:49.540542 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 22:59:51.629337 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=14,nova_api:UPDATE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:59:51.630009 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=10,nova_api:DELETE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:59:51.779942 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:59:51.780671 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell1:SELECT=11,nova_cell1:INSERT=17 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:59:53.248386 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=18,nova_cell0:UPDATE=3 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:59:53.258671 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=17,nova_cell0:UPDATE=3 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:59:59.366832 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=11,nova_api:UPDATE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:59:59.452920 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=13,nova_api:DELETE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:59:59.536456 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 22:59:59.537562 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell1:SELECT=11,nova_cell1:SAVEPOINT=1,nova_cell1:INSERT=18,nova_cell1:RELEASE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:00:15.300379 n-d-853704-1 nova-conductor[120657]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 18 23:00:23.247607 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=5,nova_cell0:UPDATE=3 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:00:33.260599 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:01:03.249220 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:01:13.261661 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:01:33.274849 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=3,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:01:33.280534 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=5,nova_cell0:UPDATE=3 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:01:38.448028 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-c06a7e81-9fab-4950-8dbb-b973c9d57a2d tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Took 0.61 seconds to select destinations for 1 instance(s). {{(pid=120657) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 18 23:01:38.471283 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c06a7e81-9fab-4950-8dbb-b973c9d57a2d tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:01:38.471772 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c06a7e81-9fab-4950-8dbb-b973c9d57a2d tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:01:38.472275 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c06a7e81-9fab-4950-8dbb-b973c9d57a2d tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:01:38.618462 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c06a7e81-9fab-4950-8dbb-b973c9d57a2d tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:01:38.618804 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c06a7e81-9fab-4950-8dbb-b973c9d57a2d tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:01:38.619191 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c06a7e81-9fab-4950-8dbb-b973c9d57a2d tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:01:38.619875 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c06a7e81-9fab-4950-8dbb-b973c9d57a2d tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:01:38.620282 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c06a7e81-9fab-4950-8dbb-b973c9d57a2d tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:01:38.620688 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c06a7e81-9fab-4950-8dbb-b973c9d57a2d tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:01:38.636124 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-c06a7e81-9fab-4950-8dbb-b973c9d57a2d tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Getting quotas for project 59dde594f2d04d40886a969d353b0bc5. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 18 23:01:38.640597 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-c06a7e81-9fab-4950-8dbb-b973c9d57a2d tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Getting quotas for user 372a102cdb374e8e8d8f0ebe31c78e58 and project 59dde594f2d04d40886a969d353b0bc5. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 18 23:01:38.652359 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-c06a7e81-9fab-4950-8dbb-b973c9d57a2d tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] [instance: 46d10e79-19a3-48bb-bbac-6b6186e68956] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120657) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 18 23:01:38.653138 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c06a7e81-9fab-4950-8dbb-b973c9d57a2d tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:01:38.653575 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c06a7e81-9fab-4950-8dbb-b973c9d57a2d tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:01:38.653963 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c06a7e81-9fab-4950-8dbb-b973c9d57a2d tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:01:38.717093 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-c06a7e81-9fab-4950-8dbb-b973c9d57a2d tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] [instance: 46d10e79-19a3-48bb-bbac-6b6186e68956] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='87b0fec9-a948-41f9-a3c0-f0a6a5a8f1e7',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120657) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 18 23:01:38.718534 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c06a7e81-9fab-4950-8dbb-b973c9d57a2d tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:01:38.718986 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c06a7e81-9fab-4950-8dbb-b973c9d57a2d tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:01:38.719391 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c06a7e81-9fab-4950-8dbb-b973c9d57a2d tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:01:38.930137 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c06a7e81-9fab-4950-8dbb-b973c9d57a2d tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:01:38.930692 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c06a7e81-9fab-4950-8dbb-b973c9d57a2d tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:01:38.931225 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c06a7e81-9fab-4950-8dbb-b973c9d57a2d tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:01:38.931750 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c06a7e81-9fab-4950-8dbb-b973c9d57a2d tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Acquiring lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 18 23:01:38.932401 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c06a7e81-9fab-4950-8dbb-b973c9d57a2d tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Acquired lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 18 23:01:38.933559 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-76b730af-449f-4bf1-9f5e-f7c6ed451eb5 None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:01:38.933951 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-76b730af-449f-4bf1-9f5e-f7c6ed451eb5 None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:01:38.934513 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-76b730af-449f-4bf1-9f5e-f7c6ed451eb5 None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:01:38.935212 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-76b730af-449f-4bf1-9f5e-f7c6ed451eb5 None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:01:38.935883 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-76b730af-449f-4bf1-9f5e-f7c6ed451eb5 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:01:38.936281 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-76b730af-449f-4bf1-9f5e-f7c6ed451eb5 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:01:38.946771 n-d-853704-1 nova-conductor[120657]: ERROR nova.compute.rpcapi [None req-76b730af-449f-4bf1-9f5e-f7c6ed451eb5 None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 18 23:01:38.947171 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-76b730af-449f-4bf1-9f5e-f7c6ed451eb5 None None] Releasing lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 18 23:01:38.948140 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server [None req-76b730af-449f-4bf1-9f5e-f7c6ed451eb5 None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 23:01:38.948140 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 23:01:38.948140 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 18 23:01:38.948140 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 18 23:01:38.948140 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 18 23:01:38.948140 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 23:01:38.948140 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 18 23:01:38.948140 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 23:01:38.948140 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 23:01:38.948140 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 18 23:01:38.948140 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 18 23:01:38.948140 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 18 23:01:38.948140 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 18 23:01:38.948140 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 18 23:01:38.948140 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 18 23:01:38.948140 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 18 23:01:38.948140 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 18 23:01:38.948140 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 18 23:01:38.948140 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 18 23:01:38.948140 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 18 23:01:38.948140 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 18 23:01:38.948140 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 18 23:01:38.948140 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 18 23:01:38.948140 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 23:01:38.948140 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 23:01:48.790478 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=9,nova_api:UPDATE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:01:48.836886 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=15,nova_api:DELETE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:01:48.942456 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:01:48.945874 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell1:SELECT=11,nova_cell1:INSERT=17 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:02:13.269181 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:02:13.276143 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=15,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:02:53.270140 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:02:53.276398 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:03:13.277591 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=3,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:03:23.271318 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=5,nova_cell0:UPDATE=3 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:03:33.286892 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=4,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:03:35.309773 n-d-853704-1 nova-conductor[120657]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 18 23:04:02.235412 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-5d454052-4947-4cd7-8ac4-9ec4dc6445ee tempest-VolumesActionsTest-588385132 tempest-VolumesActionsTest-588385132-project] Took 1.66 seconds to select destinations for 1 instance(s). {{(pid=120658) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 18 23:04:02.254813 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-5d454052-4947-4cd7-8ac4-9ec4dc6445ee tempest-VolumesActionsTest-588385132 tempest-VolumesActionsTest-588385132-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:04:02.255248 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-5d454052-4947-4cd7-8ac4-9ec4dc6445ee tempest-VolumesActionsTest-588385132 tempest-VolumesActionsTest-588385132-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:04:02.255640 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-5d454052-4947-4cd7-8ac4-9ec4dc6445ee tempest-VolumesActionsTest-588385132 tempest-VolumesActionsTest-588385132-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:04:02.439439 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:04:02.441540 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-5d454052-4947-4cd7-8ac4-9ec4dc6445ee tempest-VolumesActionsTest-588385132 tempest-VolumesActionsTest-588385132-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:04:02.441976 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-5d454052-4947-4cd7-8ac4-9ec4dc6445ee tempest-VolumesActionsTest-588385132 tempest-VolumesActionsTest-588385132-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:04:02.442374 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-5d454052-4947-4cd7-8ac4-9ec4dc6445ee tempest-VolumesActionsTest-588385132 tempest-VolumesActionsTest-588385132-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:04:02.443032 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-5d454052-4947-4cd7-8ac4-9ec4dc6445ee tempest-VolumesActionsTest-588385132 tempest-VolumesActionsTest-588385132-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:04:02.443448 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-5d454052-4947-4cd7-8ac4-9ec4dc6445ee tempest-VolumesActionsTest-588385132 tempest-VolumesActionsTest-588385132-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:04:02.443861 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-5d454052-4947-4cd7-8ac4-9ec4dc6445ee tempest-VolumesActionsTest-588385132 tempest-VolumesActionsTest-588385132-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:04:02.459276 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-5d454052-4947-4cd7-8ac4-9ec4dc6445ee tempest-VolumesActionsTest-588385132 tempest-VolumesActionsTest-588385132-project] Getting quotas for project 13615f0095934a49ab31766b782ebcf7. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 18 23:04:02.463645 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-5d454052-4947-4cd7-8ac4-9ec4dc6445ee tempest-VolumesActionsTest-588385132 tempest-VolumesActionsTest-588385132-project] Getting quotas for user 65afe546bc844036824db23c5abd88dd and project 13615f0095934a49ab31766b782ebcf7. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 18 23:04:02.475325 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-5d454052-4947-4cd7-8ac4-9ec4dc6445ee tempest-VolumesActionsTest-588385132 tempest-VolumesActionsTest-588385132-project] [instance: 6290459e-69b5-4697-a168-e3bb6c08ad05] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120658) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 18 23:04:02.484080 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-5d454052-4947-4cd7-8ac4-9ec4dc6445ee tempest-VolumesActionsTest-588385132 tempest-VolumesActionsTest-588385132-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:04:02.484080 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-5d454052-4947-4cd7-8ac4-9ec4dc6445ee tempest-VolumesActionsTest-588385132 tempest-VolumesActionsTest-588385132-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:04:02.484080 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-5d454052-4947-4cd7-8ac4-9ec4dc6445ee tempest-VolumesActionsTest-588385132 tempest-VolumesActionsTest-588385132-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:04:02.548337 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-5d454052-4947-4cd7-8ac4-9ec4dc6445ee tempest-VolumesActionsTest-588385132 tempest-VolumesActionsTest-588385132-project] [instance: 6290459e-69b5-4697-a168-e3bb6c08ad05] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='87b0fec9-a948-41f9-a3c0-f0a6a5a8f1e7',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120658) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 18 23:04:02.549542 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-5d454052-4947-4cd7-8ac4-9ec4dc6445ee tempest-VolumesActionsTest-588385132 tempest-VolumesActionsTest-588385132-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:04:02.549980 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-5d454052-4947-4cd7-8ac4-9ec4dc6445ee tempest-VolumesActionsTest-588385132 tempest-VolumesActionsTest-588385132-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:04:02.550378 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-5d454052-4947-4cd7-8ac4-9ec4dc6445ee tempest-VolumesActionsTest-588385132 tempest-VolumesActionsTest-588385132-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:04:02.804243 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-5d454052-4947-4cd7-8ac4-9ec4dc6445ee tempest-VolumesActionsTest-588385132 tempest-VolumesActionsTest-588385132-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:04:02.804692 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-5d454052-4947-4cd7-8ac4-9ec4dc6445ee tempest-VolumesActionsTest-588385132 tempest-VolumesActionsTest-588385132-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:04:02.805115 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-5d454052-4947-4cd7-8ac4-9ec4dc6445ee tempest-VolumesActionsTest-588385132 tempest-VolumesActionsTest-588385132-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:04:02.805561 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-5d454052-4947-4cd7-8ac4-9ec4dc6445ee tempest-VolumesActionsTest-588385132 tempest-VolumesActionsTest-588385132-project] Acquiring lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 18 23:04:02.805908 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-5d454052-4947-4cd7-8ac4-9ec4dc6445ee tempest-VolumesActionsTest-588385132 tempest-VolumesActionsTest-588385132-project] Acquired lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 18 23:04:02.806667 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-e729a218-0738-458b-934b-9266731e1ffb None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:04:02.807077 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-e729a218-0738-458b-934b-9266731e1ffb None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:04:02.807457 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-e729a218-0738-458b-934b-9266731e1ffb None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:04:02.808136 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-e729a218-0738-458b-934b-9266731e1ffb None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:04:02.808548 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-e729a218-0738-458b-934b-9266731e1ffb None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:04:02.808943 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-e729a218-0738-458b-934b-9266731e1ffb None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:04:02.818381 n-d-853704-1 nova-conductor[120658]: ERROR nova.compute.rpcapi [None req-e729a218-0738-458b-934b-9266731e1ffb None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 18 23:04:02.819822 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-e729a218-0738-458b-934b-9266731e1ffb None None] Releasing lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 18 23:04:02.819822 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server [None req-e729a218-0738-458b-934b-9266731e1ffb None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 23:04:02.819822 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 23:04:02.819822 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 18 23:04:02.819822 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 18 23:04:02.819822 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 18 23:04:02.819822 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 23:04:02.819822 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 18 23:04:02.819822 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 23:04:02.819822 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 23:04:02.819822 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 18 23:04:02.819822 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 18 23:04:02.819822 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 18 23:04:02.819822 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 18 23:04:02.819822 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 18 23:04:02.819822 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 18 23:04:02.819822 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 18 23:04:02.819822 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 18 23:04:02.819822 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 18 23:04:02.819822 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 18 23:04:02.819822 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 18 23:04:02.819822 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 18 23:04:02.819822 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 18 23:04:02.819822 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 18 23:04:02.819822 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 23:04:02.819822 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 23:04:03.296589 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=6,nova_cell0:UPDATE=3 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:04:12.724924 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=12,nova_api:UPDATE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:04:12.726137 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=12,nova_api:DELETE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:04:12.816138 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:04:12.817315 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell1:SELECT=11,nova_cell1:SAVEPOINT=1,nova_cell1:INSERT=18,nova_cell1:RELEASE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:04:14.672122 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-429e4dbc-f1ea-4743-a885-8fb37691ef39 tempest-ServerMetadataNegativeTestJSON-708979680 tempest-ServerMetadataNegativeTestJSON-708979680-project] Took 0.57 seconds to select destinations for 1 instance(s). {{(pid=120657) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 18 23:04:14.703433 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-429e4dbc-f1ea-4743-a885-8fb37691ef39 tempest-ServerMetadataNegativeTestJSON-708979680 tempest-ServerMetadataNegativeTestJSON-708979680-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:04:14.703433 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-429e4dbc-f1ea-4743-a885-8fb37691ef39 tempest-ServerMetadataNegativeTestJSON-708979680 tempest-ServerMetadataNegativeTestJSON-708979680-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:04:14.703433 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-429e4dbc-f1ea-4743-a885-8fb37691ef39 tempest-ServerMetadataNegativeTestJSON-708979680 tempest-ServerMetadataNegativeTestJSON-708979680-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:04:14.894720 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-429e4dbc-f1ea-4743-a885-8fb37691ef39 tempest-ServerMetadataNegativeTestJSON-708979680 tempest-ServerMetadataNegativeTestJSON-708979680-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:04:14.898041 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-429e4dbc-f1ea-4743-a885-8fb37691ef39 tempest-ServerMetadataNegativeTestJSON-708979680 tempest-ServerMetadataNegativeTestJSON-708979680-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:04:14.898041 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-429e4dbc-f1ea-4743-a885-8fb37691ef39 tempest-ServerMetadataNegativeTestJSON-708979680 tempest-ServerMetadataNegativeTestJSON-708979680-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:04:14.898041 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-429e4dbc-f1ea-4743-a885-8fb37691ef39 tempest-ServerMetadataNegativeTestJSON-708979680 tempest-ServerMetadataNegativeTestJSON-708979680-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:04:14.898041 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-429e4dbc-f1ea-4743-a885-8fb37691ef39 tempest-ServerMetadataNegativeTestJSON-708979680 tempest-ServerMetadataNegativeTestJSON-708979680-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:04:14.898041 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-429e4dbc-f1ea-4743-a885-8fb37691ef39 tempest-ServerMetadataNegativeTestJSON-708979680 tempest-ServerMetadataNegativeTestJSON-708979680-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:04:14.913063 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-429e4dbc-f1ea-4743-a885-8fb37691ef39 tempest-ServerMetadataNegativeTestJSON-708979680 tempest-ServerMetadataNegativeTestJSON-708979680-project] Getting quotas for project 2fbfb398f9ee468fa22c737d559e8557. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 18 23:04:14.918272 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-429e4dbc-f1ea-4743-a885-8fb37691ef39 tempest-ServerMetadataNegativeTestJSON-708979680 tempest-ServerMetadataNegativeTestJSON-708979680-project] Getting quotas for user 9f42e3f683f24423b09d8ca04531a9b8 and project 2fbfb398f9ee468fa22c737d559e8557. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 18 23:04:14.930064 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-429e4dbc-f1ea-4743-a885-8fb37691ef39 tempest-ServerMetadataNegativeTestJSON-708979680 tempest-ServerMetadataNegativeTestJSON-708979680-project] [instance: 1b6bfbe6-966b-4f30-8121-94137b38bcd1] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120657) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 18 23:04:14.930846 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-429e4dbc-f1ea-4743-a885-8fb37691ef39 tempest-ServerMetadataNegativeTestJSON-708979680 tempest-ServerMetadataNegativeTestJSON-708979680-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:04:14.931285 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-429e4dbc-f1ea-4743-a885-8fb37691ef39 tempest-ServerMetadataNegativeTestJSON-708979680 tempest-ServerMetadataNegativeTestJSON-708979680-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:04:14.931699 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-429e4dbc-f1ea-4743-a885-8fb37691ef39 tempest-ServerMetadataNegativeTestJSON-708979680 tempest-ServerMetadataNegativeTestJSON-708979680-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:04:14.999910 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-429e4dbc-f1ea-4743-a885-8fb37691ef39 tempest-ServerMetadataNegativeTestJSON-708979680 tempest-ServerMetadataNegativeTestJSON-708979680-project] [instance: 1b6bfbe6-966b-4f30-8121-94137b38bcd1] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='87b0fec9-a948-41f9-a3c0-f0a6a5a8f1e7',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120657) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 18 23:04:15.001229 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-429e4dbc-f1ea-4743-a885-8fb37691ef39 tempest-ServerMetadataNegativeTestJSON-708979680 tempest-ServerMetadataNegativeTestJSON-708979680-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:04:15.001681 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-429e4dbc-f1ea-4743-a885-8fb37691ef39 tempest-ServerMetadataNegativeTestJSON-708979680 tempest-ServerMetadataNegativeTestJSON-708979680-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:04:15.003793 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-429e4dbc-f1ea-4743-a885-8fb37691ef39 tempest-ServerMetadataNegativeTestJSON-708979680 tempest-ServerMetadataNegativeTestJSON-708979680-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:04:15.212083 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-429e4dbc-f1ea-4743-a885-8fb37691ef39 tempest-ServerMetadataNegativeTestJSON-708979680 tempest-ServerMetadataNegativeTestJSON-708979680-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:04:15.212531 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-429e4dbc-f1ea-4743-a885-8fb37691ef39 tempest-ServerMetadataNegativeTestJSON-708979680 tempest-ServerMetadataNegativeTestJSON-708979680-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:04:15.212931 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-429e4dbc-f1ea-4743-a885-8fb37691ef39 tempest-ServerMetadataNegativeTestJSON-708979680 tempest-ServerMetadataNegativeTestJSON-708979680-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:04:15.213601 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-429e4dbc-f1ea-4743-a885-8fb37691ef39 tempest-ServerMetadataNegativeTestJSON-708979680 tempest-ServerMetadataNegativeTestJSON-708979680-project] Acquiring lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 18 23:04:15.213803 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-429e4dbc-f1ea-4743-a885-8fb37691ef39 tempest-ServerMetadataNegativeTestJSON-708979680 tempest-ServerMetadataNegativeTestJSON-708979680-project] Acquired lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 18 23:04:15.214474 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-894a2a1f-e954-4b30-9a83-d1387d073d28 None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:04:15.215044 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-894a2a1f-e954-4b30-9a83-d1387d073d28 None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:04:15.215461 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-894a2a1f-e954-4b30-9a83-d1387d073d28 None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:04:15.216285 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-894a2a1f-e954-4b30-9a83-d1387d073d28 None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:04:15.216697 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-894a2a1f-e954-4b30-9a83-d1387d073d28 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:04:15.217128 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-894a2a1f-e954-4b30-9a83-d1387d073d28 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:04:15.227635 n-d-853704-1 nova-conductor[120657]: ERROR nova.compute.rpcapi [None req-894a2a1f-e954-4b30-9a83-d1387d073d28 None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 18 23:04:15.228050 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-894a2a1f-e954-4b30-9a83-d1387d073d28 None None] Releasing lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 18 23:04:15.228968 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server [None req-894a2a1f-e954-4b30-9a83-d1387d073d28 None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 23:04:15.228968 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 23:04:15.228968 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 18 23:04:15.228968 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 18 23:04:15.228968 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 18 23:04:15.228968 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 23:04:15.228968 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 18 23:04:15.228968 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 23:04:15.228968 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 23:04:15.228968 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 18 23:04:15.228968 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 18 23:04:15.228968 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 18 23:04:15.228968 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 18 23:04:15.228968 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 18 23:04:15.228968 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 18 23:04:15.228968 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 18 23:04:15.228968 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 18 23:04:15.228968 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 18 23:04:15.228968 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 18 23:04:15.228968 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 18 23:04:15.228968 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 18 23:04:15.228968 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 18 23:04:15.228968 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 18 23:04:15.228968 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 23:04:15.228968 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 23:04:25.090359 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=12,nova_api:UPDATE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:04:25.163310 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=12,nova_api:DELETE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:04:25.227045 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:04:25.227828 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell1:SELECT=11,nova_cell1:SAVEPOINT=1,nova_cell1:INSERT=18,nova_cell1:RELEASE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:04:33.287473 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=12,nova_cell0:UPDATE=3 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:04:43.307895 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=15,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:04:53.291015 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=3,nova_cell0:UPDATE=2 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:05:23.312005 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=7,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:05:23.776966 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-7f91206e-be96-4fb6-9756-73fb50a89d24 tempest-MultipleCreateTestJSON-1319949902 tempest-MultipleCreateTestJSON-1319949902-project] Took 0.64 seconds to select destinations for 2 instance(s). {{(pid=120658) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 18 23:05:23.797888 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-7f91206e-be96-4fb6-9756-73fb50a89d24 tempest-MultipleCreateTestJSON-1319949902 tempest-MultipleCreateTestJSON-1319949902-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:05:23.798329 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-7f91206e-be96-4fb6-9756-73fb50a89d24 tempest-MultipleCreateTestJSON-1319949902 tempest-MultipleCreateTestJSON-1319949902-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:05:23.801485 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-7f91206e-be96-4fb6-9756-73fb50a89d24 tempest-MultipleCreateTestJSON-1319949902 tempest-MultipleCreateTestJSON-1319949902-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:05:23.938967 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-7f91206e-be96-4fb6-9756-73fb50a89d24 tempest-MultipleCreateTestJSON-1319949902 tempest-MultipleCreateTestJSON-1319949902-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:05:23.939967 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-7f91206e-be96-4fb6-9756-73fb50a89d24 tempest-MultipleCreateTestJSON-1319949902 tempest-MultipleCreateTestJSON-1319949902-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:05:23.940761 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-7f91206e-be96-4fb6-9756-73fb50a89d24 tempest-MultipleCreateTestJSON-1319949902 tempest-MultipleCreateTestJSON-1319949902-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:05:24.079232 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:05:24.085415 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-7f91206e-be96-4fb6-9756-73fb50a89d24 tempest-MultipleCreateTestJSON-1319949902 tempest-MultipleCreateTestJSON-1319949902-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:05:24.085894 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-7f91206e-be96-4fb6-9756-73fb50a89d24 tempest-MultipleCreateTestJSON-1319949902 tempest-MultipleCreateTestJSON-1319949902-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.002s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:05:24.086333 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-7f91206e-be96-4fb6-9756-73fb50a89d24 tempest-MultipleCreateTestJSON-1319949902 tempest-MultipleCreateTestJSON-1319949902-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:05:24.087361 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-7f91206e-be96-4fb6-9756-73fb50a89d24 tempest-MultipleCreateTestJSON-1319949902 tempest-MultipleCreateTestJSON-1319949902-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:05:24.087807 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-7f91206e-be96-4fb6-9756-73fb50a89d24 tempest-MultipleCreateTestJSON-1319949902 tempest-MultipleCreateTestJSON-1319949902-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:05:24.088361 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-7f91206e-be96-4fb6-9756-73fb50a89d24 tempest-MultipleCreateTestJSON-1319949902 tempest-MultipleCreateTestJSON-1319949902-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:05:24.106235 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-7f91206e-be96-4fb6-9756-73fb50a89d24 tempest-MultipleCreateTestJSON-1319949902 tempest-MultipleCreateTestJSON-1319949902-project] Getting quotas for project 9b1d3d3bc3f0421781c885492c70406d. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 18 23:05:24.112276 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-7f91206e-be96-4fb6-9756-73fb50a89d24 tempest-MultipleCreateTestJSON-1319949902 tempest-MultipleCreateTestJSON-1319949902-project] Getting quotas for user 618be23c9d424627a3052447bfe8cf43 and project 9b1d3d3bc3f0421781c885492c70406d. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 18 23:05:24.129312 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-7f91206e-be96-4fb6-9756-73fb50a89d24 tempest-MultipleCreateTestJSON-1319949902 tempest-MultipleCreateTestJSON-1319949902-project] [instance: ffb455a3-e582-4abd-91b4-1b38f4547404] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [] {{(pid=120658) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 18 23:05:24.136914 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-7f91206e-be96-4fb6-9756-73fb50a89d24 tempest-MultipleCreateTestJSON-1319949902 tempest-MultipleCreateTestJSON-1319949902-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:05:24.136914 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-7f91206e-be96-4fb6-9756-73fb50a89d24 tempest-MultipleCreateTestJSON-1319949902 tempest-MultipleCreateTestJSON-1319949902-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:05:24.136914 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-7f91206e-be96-4fb6-9756-73fb50a89d24 tempest-MultipleCreateTestJSON-1319949902 tempest-MultipleCreateTestJSON-1319949902-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:05:24.205395 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-7f91206e-be96-4fb6-9756-73fb50a89d24 tempest-MultipleCreateTestJSON-1319949902 tempest-MultipleCreateTestJSON-1319949902-project] [instance: ffb455a3-e582-4abd-91b4-1b38f4547404] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='87b0fec9-a948-41f9-a3c0-f0a6a5a8f1e7',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120658) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 18 23:05:24.206934 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-7f91206e-be96-4fb6-9756-73fb50a89d24 tempest-MultipleCreateTestJSON-1319949902 tempest-MultipleCreateTestJSON-1319949902-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:05:24.207584 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-7f91206e-be96-4fb6-9756-73fb50a89d24 tempest-MultipleCreateTestJSON-1319949902 tempest-MultipleCreateTestJSON-1319949902-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:05:24.208173 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-7f91206e-be96-4fb6-9756-73fb50a89d24 tempest-MultipleCreateTestJSON-1319949902 tempest-MultipleCreateTestJSON-1319949902-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:05:24.442256 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-7f91206e-be96-4fb6-9756-73fb50a89d24 tempest-MultipleCreateTestJSON-1319949902 tempest-MultipleCreateTestJSON-1319949902-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:05:24.442754 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-7f91206e-be96-4fb6-9756-73fb50a89d24 tempest-MultipleCreateTestJSON-1319949902 tempest-MultipleCreateTestJSON-1319949902-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:05:24.443204 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-7f91206e-be96-4fb6-9756-73fb50a89d24 tempest-MultipleCreateTestJSON-1319949902 tempest-MultipleCreateTestJSON-1319949902-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:05:24.443749 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-7f91206e-be96-4fb6-9756-73fb50a89d24 tempest-MultipleCreateTestJSON-1319949902 tempest-MultipleCreateTestJSON-1319949902-project] Acquiring lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 18 23:05:24.444192 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-7f91206e-be96-4fb6-9756-73fb50a89d24 tempest-MultipleCreateTestJSON-1319949902 tempest-MultipleCreateTestJSON-1319949902-project] Acquired lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 18 23:05:24.445004 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ebf181b9-bdce-4710-9884-7f05732bf171 None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:05:24.445459 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ebf181b9-bdce-4710-9884-7f05732bf171 None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:05:24.445887 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ebf181b9-bdce-4710-9884-7f05732bf171 None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:05:24.446676 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ebf181b9-bdce-4710-9884-7f05732bf171 None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:05:24.447136 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ebf181b9-bdce-4710-9884-7f05732bf171 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:05:24.447577 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ebf181b9-bdce-4710-9884-7f05732bf171 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:05:24.458821 n-d-853704-1 nova-conductor[120658]: ERROR nova.compute.rpcapi [None req-ebf181b9-bdce-4710-9884-7f05732bf171 None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 18 23:05:24.459481 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ebf181b9-bdce-4710-9884-7f05732bf171 None None] Releasing lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 18 23:05:24.460742 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server [None req-ebf181b9-bdce-4710-9884-7f05732bf171 None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 23:05:24.460742 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 23:05:24.460742 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 18 23:05:24.460742 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 18 23:05:24.460742 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 18 23:05:24.460742 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 23:05:24.460742 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 18 23:05:24.460742 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 23:05:24.460742 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 23:05:24.460742 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 18 23:05:24.460742 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 18 23:05:24.460742 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 18 23:05:24.460742 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 18 23:05:24.460742 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 18 23:05:24.460742 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 18 23:05:24.460742 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 18 23:05:24.460742 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 18 23:05:24.460742 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 18 23:05:24.460742 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 18 23:05:24.460742 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 18 23:05:24.460742 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 18 23:05:24.460742 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 18 23:05:24.460742 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 18 23:05:24.460742 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 23:05:24.460742 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 23:05:34.366452 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=17 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:05:34.367293 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=13,nova_api:UPDATE=1,nova_api:DELETE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:05:34.453348 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:05:34.456118 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell1:SELECT=16,nova_cell1:INSERT=32 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:05:43.473545 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-26c6fa0b-8d79-4608-bb0c-fb3e3414abef tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Took 0.44 seconds to select destinations for 1 instance(s). {{(pid=120657) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 18 23:05:43.493405 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-26c6fa0b-8d79-4608-bb0c-fb3e3414abef tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:05:43.493844 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-26c6fa0b-8d79-4608-bb0c-fb3e3414abef tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:05:43.494240 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-26c6fa0b-8d79-4608-bb0c-fb3e3414abef tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:05:43.599773 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-26c6fa0b-8d79-4608-bb0c-fb3e3414abef tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:05:43.600150 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-26c6fa0b-8d79-4608-bb0c-fb3e3414abef tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:05:43.600547 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-26c6fa0b-8d79-4608-bb0c-fb3e3414abef tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:05:43.601215 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-26c6fa0b-8d79-4608-bb0c-fb3e3414abef tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:05:43.601632 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-26c6fa0b-8d79-4608-bb0c-fb3e3414abef tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:05:43.602025 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-26c6fa0b-8d79-4608-bb0c-fb3e3414abef tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:05:43.618475 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-26c6fa0b-8d79-4608-bb0c-fb3e3414abef tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Getting quotas for project 05baaf842a13444d8905683625e5853a. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 18 23:05:43.622944 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-26c6fa0b-8d79-4608-bb0c-fb3e3414abef tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Getting quotas for user 4d73c623dd454555ac8d974cf7a60398 and project 05baaf842a13444d8905683625e5853a. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 18 23:05:43.634130 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-26c6fa0b-8d79-4608-bb0c-fb3e3414abef tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] [instance: 1de29093-70ea-45e7-91fe-4c5fb152c02c] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120657) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 18 23:05:43.634889 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-26c6fa0b-8d79-4608-bb0c-fb3e3414abef tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:05:43.635323 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-26c6fa0b-8d79-4608-bb0c-fb3e3414abef tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:05:43.635746 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-26c6fa0b-8d79-4608-bb0c-fb3e3414abef tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:05:43.697973 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-26c6fa0b-8d79-4608-bb0c-fb3e3414abef tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] [instance: 1de29093-70ea-45e7-91fe-4c5fb152c02c] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='87b0fec9-a948-41f9-a3c0-f0a6a5a8f1e7',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120657) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 18 23:05:43.699196 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-26c6fa0b-8d79-4608-bb0c-fb3e3414abef tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:05:43.699637 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-26c6fa0b-8d79-4608-bb0c-fb3e3414abef tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:05:43.700097 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-26c6fa0b-8d79-4608-bb0c-fb3e3414abef tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:05:43.927036 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-26c6fa0b-8d79-4608-bb0c-fb3e3414abef tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:05:43.927395 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-26c6fa0b-8d79-4608-bb0c-fb3e3414abef tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:05:43.927819 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-26c6fa0b-8d79-4608-bb0c-fb3e3414abef tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:05:43.928283 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-26c6fa0b-8d79-4608-bb0c-fb3e3414abef tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Acquiring lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 18 23:05:43.928613 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-26c6fa0b-8d79-4608-bb0c-fb3e3414abef tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Acquired lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 18 23:05:43.929330 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-56c84a7a-d076-4983-89cb-146fc4bf27fb None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:05:43.929736 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-56c84a7a-d076-4983-89cb-146fc4bf27fb None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:05:43.930143 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-56c84a7a-d076-4983-89cb-146fc4bf27fb None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:05:43.930776 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-56c84a7a-d076-4983-89cb-146fc4bf27fb None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:05:43.931185 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-56c84a7a-d076-4983-89cb-146fc4bf27fb None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:05:43.931611 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-56c84a7a-d076-4983-89cb-146fc4bf27fb None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:05:43.939883 n-d-853704-1 nova-conductor[120657]: ERROR nova.compute.rpcapi [None req-56c84a7a-d076-4983-89cb-146fc4bf27fb None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 18 23:05:43.940268 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-56c84a7a-d076-4983-89cb-146fc4bf27fb None None] Releasing lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 18 23:05:43.941154 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server [None req-56c84a7a-d076-4983-89cb-146fc4bf27fb None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 23:05:43.941154 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 23:05:43.941154 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 18 23:05:43.941154 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 18 23:05:43.941154 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 18 23:05:43.941154 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 23:05:43.941154 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 18 23:05:43.941154 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 23:05:43.941154 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 23:05:43.941154 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 18 23:05:43.941154 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 18 23:05:43.941154 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 18 23:05:43.941154 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 18 23:05:43.941154 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 18 23:05:43.941154 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 18 23:05:43.941154 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 18 23:05:43.941154 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 18 23:05:43.941154 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 18 23:05:43.941154 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 18 23:05:43.941154 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 18 23:05:43.941154 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 18 23:05:43.941154 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 18 23:05:43.941154 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 18 23:05:43.941154 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 23:05:43.941154 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 23:05:53.853527 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=12 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:05:53.854725 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=12,nova_api:UPDATE=1,nova_api:DELETE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:05:53.937361 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:05:53.942988 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell1:SELECT=11,nova_cell1:INSERT=17 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:05:59.115727 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-90c36682-6af7-40b0-a0e0-55778d17255e tempest-VolumesSnapshotTestJSON-1490689032 tempest-VolumesSnapshotTestJSON-1490689032-project] Took 0.43 seconds to select destinations for 1 instance(s). {{(pid=120658) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 18 23:05:59.137006 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-90c36682-6af7-40b0-a0e0-55778d17255e tempest-VolumesSnapshotTestJSON-1490689032 tempest-VolumesSnapshotTestJSON-1490689032-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:05:59.137541 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-90c36682-6af7-40b0-a0e0-55778d17255e tempest-VolumesSnapshotTestJSON-1490689032 tempest-VolumesSnapshotTestJSON-1490689032-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:05:59.138004 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-90c36682-6af7-40b0-a0e0-55778d17255e tempest-VolumesSnapshotTestJSON-1490689032 tempest-VolumesSnapshotTestJSON-1490689032-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:05:59.253706 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=12,nova_cell0:UPDATE=3 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:05:59.256850 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-90c36682-6af7-40b0-a0e0-55778d17255e tempest-VolumesSnapshotTestJSON-1490689032 tempest-VolumesSnapshotTestJSON-1490689032-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:05:59.257282 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-90c36682-6af7-40b0-a0e0-55778d17255e tempest-VolumesSnapshotTestJSON-1490689032 tempest-VolumesSnapshotTestJSON-1490689032-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:05:59.257666 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-90c36682-6af7-40b0-a0e0-55778d17255e tempest-VolumesSnapshotTestJSON-1490689032 tempest-VolumesSnapshotTestJSON-1490689032-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:05:59.258314 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-90c36682-6af7-40b0-a0e0-55778d17255e tempest-VolumesSnapshotTestJSON-1490689032 tempest-VolumesSnapshotTestJSON-1490689032-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:05:59.258720 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-90c36682-6af7-40b0-a0e0-55778d17255e tempest-VolumesSnapshotTestJSON-1490689032 tempest-VolumesSnapshotTestJSON-1490689032-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:05:59.259094 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-90c36682-6af7-40b0-a0e0-55778d17255e tempest-VolumesSnapshotTestJSON-1490689032 tempest-VolumesSnapshotTestJSON-1490689032-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:05:59.274516 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-90c36682-6af7-40b0-a0e0-55778d17255e tempest-VolumesSnapshotTestJSON-1490689032 tempest-VolumesSnapshotTestJSON-1490689032-project] Getting quotas for project 6ab620df4d7d41738193bb1eb0ad93a1. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 18 23:05:59.278863 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-90c36682-6af7-40b0-a0e0-55778d17255e tempest-VolumesSnapshotTestJSON-1490689032 tempest-VolumesSnapshotTestJSON-1490689032-project] Getting quotas for user 333a292527ac4188aaa4a20f192eb9a6 and project 6ab620df4d7d41738193bb1eb0ad93a1. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 18 23:05:59.290376 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-90c36682-6af7-40b0-a0e0-55778d17255e tempest-VolumesSnapshotTestJSON-1490689032 tempest-VolumesSnapshotTestJSON-1490689032-project] [instance: e7320e25-20dc-457b-beb9-e2b506e50592] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120658) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 18 23:05:59.291117 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-90c36682-6af7-40b0-a0e0-55778d17255e tempest-VolumesSnapshotTestJSON-1490689032 tempest-VolumesSnapshotTestJSON-1490689032-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:05:59.291535 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-90c36682-6af7-40b0-a0e0-55778d17255e tempest-VolumesSnapshotTestJSON-1490689032 tempest-VolumesSnapshotTestJSON-1490689032-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:05:59.291954 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-90c36682-6af7-40b0-a0e0-55778d17255e tempest-VolumesSnapshotTestJSON-1490689032 tempest-VolumesSnapshotTestJSON-1490689032-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:05:59.354494 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-90c36682-6af7-40b0-a0e0-55778d17255e tempest-VolumesSnapshotTestJSON-1490689032 tempest-VolumesSnapshotTestJSON-1490689032-project] [instance: e7320e25-20dc-457b-beb9-e2b506e50592] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='87b0fec9-a948-41f9-a3c0-f0a6a5a8f1e7',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120658) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 18 23:05:59.355725 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-90c36682-6af7-40b0-a0e0-55778d17255e tempest-VolumesSnapshotTestJSON-1490689032 tempest-VolumesSnapshotTestJSON-1490689032-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:05:59.356159 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-90c36682-6af7-40b0-a0e0-55778d17255e tempest-VolumesSnapshotTestJSON-1490689032 tempest-VolumesSnapshotTestJSON-1490689032-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:05:59.356555 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-90c36682-6af7-40b0-a0e0-55778d17255e tempest-VolumesSnapshotTestJSON-1490689032 tempest-VolumesSnapshotTestJSON-1490689032-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:05:59.661104 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-90c36682-6af7-40b0-a0e0-55778d17255e tempest-VolumesSnapshotTestJSON-1490689032 tempest-VolumesSnapshotTestJSON-1490689032-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:05:59.661541 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-90c36682-6af7-40b0-a0e0-55778d17255e tempest-VolumesSnapshotTestJSON-1490689032 tempest-VolumesSnapshotTestJSON-1490689032-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:05:59.661932 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-90c36682-6af7-40b0-a0e0-55778d17255e tempest-VolumesSnapshotTestJSON-1490689032 tempest-VolumesSnapshotTestJSON-1490689032-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:05:59.662379 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-90c36682-6af7-40b0-a0e0-55778d17255e tempest-VolumesSnapshotTestJSON-1490689032 tempest-VolumesSnapshotTestJSON-1490689032-project] Acquiring lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 18 23:05:59.662878 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-90c36682-6af7-40b0-a0e0-55778d17255e tempest-VolumesSnapshotTestJSON-1490689032 tempest-VolumesSnapshotTestJSON-1490689032-project] Acquired lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 18 23:05:59.663612 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-d1131f93-aecd-4789-929c-e3a420a5360e None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:05:59.664048 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-d1131f93-aecd-4789-929c-e3a420a5360e None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:05:59.664429 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-d1131f93-aecd-4789-929c-e3a420a5360e None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:05:59.665069 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-d1131f93-aecd-4789-929c-e3a420a5360e None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:05:59.665473 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-d1131f93-aecd-4789-929c-e3a420a5360e None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:05:59.665883 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-d1131f93-aecd-4789-929c-e3a420a5360e None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:05:59.674145 n-d-853704-1 nova-conductor[120658]: ERROR nova.compute.rpcapi [None req-d1131f93-aecd-4789-929c-e3a420a5360e None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 18 23:05:59.674532 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-d1131f93-aecd-4789-929c-e3a420a5360e None None] Releasing lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 18 23:05:59.675421 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server [None req-d1131f93-aecd-4789-929c-e3a420a5360e None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 23:05:59.675421 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 23:05:59.675421 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 18 23:05:59.675421 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 18 23:05:59.675421 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 18 23:05:59.675421 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 23:05:59.675421 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 18 23:05:59.675421 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 23:05:59.675421 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 23:05:59.675421 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 18 23:05:59.675421 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 18 23:05:59.675421 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 18 23:05:59.675421 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 18 23:05:59.675421 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 18 23:05:59.675421 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 18 23:05:59.675421 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 18 23:05:59.675421 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 18 23:05:59.675421 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 18 23:05:59.675421 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 18 23:05:59.675421 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 18 23:05:59.675421 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 18 23:05:59.675421 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 18 23:05:59.675421 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 18 23:05:59.675421 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 23:05:59.675421 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 23:06:03.318135 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=15,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:06:09.562188 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=12,nova_api:UPDATE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:06:09.563068 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=12,nova_api:DELETE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:06:09.672671 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:06:09.673252 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell1:SELECT=11,nova_cell1:INSERT=17 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:06:13.293814 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=7,nova_cell0:UPDATE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:06:43.329435 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:06:53.301660 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:06:55.319868 n-d-853704-1 nova-conductor[120657]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 18 23:07:03.338893 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=3,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:07:13.309851 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=4,nova_cell0:UPDATE=2 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:07:23.345840 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=4,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:07:40.870962 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-c01bece9-759c-45b3-a39f-3f5cb00dc4e5 tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Took 0.63 seconds to select destinations for 1 instance(s). {{(pid=120657) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 18 23:07:40.889933 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c01bece9-759c-45b3-a39f-3f5cb00dc4e5 tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:07:40.890372 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c01bece9-759c-45b3-a39f-3f5cb00dc4e5 tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:07:40.890764 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c01bece9-759c-45b3-a39f-3f5cb00dc4e5 tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:07:40.994413 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c01bece9-759c-45b3-a39f-3f5cb00dc4e5 tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:07:40.994845 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c01bece9-759c-45b3-a39f-3f5cb00dc4e5 tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:07:40.995228 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c01bece9-759c-45b3-a39f-3f5cb00dc4e5 tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:07:40.995949 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c01bece9-759c-45b3-a39f-3f5cb00dc4e5 tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:07:40.996393 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c01bece9-759c-45b3-a39f-3f5cb00dc4e5 tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:07:40.996770 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c01bece9-759c-45b3-a39f-3f5cb00dc4e5 tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:07:41.012099 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-c01bece9-759c-45b3-a39f-3f5cb00dc4e5 tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Getting quotas for project 59dde594f2d04d40886a969d353b0bc5. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 18 23:07:41.017189 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-c01bece9-759c-45b3-a39f-3f5cb00dc4e5 tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Getting quotas for user 372a102cdb374e8e8d8f0ebe31c78e58 and project 59dde594f2d04d40886a969d353b0bc5. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 18 23:07:41.028449 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-c01bece9-759c-45b3-a39f-3f5cb00dc4e5 tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] [instance: e761281e-4c4a-4299-ac00-444cb7e3fdda] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120657) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 18 23:07:41.030564 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c01bece9-759c-45b3-a39f-3f5cb00dc4e5 tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:07:41.030564 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c01bece9-759c-45b3-a39f-3f5cb00dc4e5 tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:07:41.030564 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c01bece9-759c-45b3-a39f-3f5cb00dc4e5 tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:07:41.092950 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-c01bece9-759c-45b3-a39f-3f5cb00dc4e5 tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] [instance: e761281e-4c4a-4299-ac00-444cb7e3fdda] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='87b0fec9-a948-41f9-a3c0-f0a6a5a8f1e7',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120657) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 18 23:07:41.094148 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c01bece9-759c-45b3-a39f-3f5cb00dc4e5 tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:07:41.095270 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c01bece9-759c-45b3-a39f-3f5cb00dc4e5 tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:07:41.095653 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c01bece9-759c-45b3-a39f-3f5cb00dc4e5 tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:07:41.339419 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c01bece9-759c-45b3-a39f-3f5cb00dc4e5 tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:07:41.339888 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c01bece9-759c-45b3-a39f-3f5cb00dc4e5 tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:07:41.340291 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c01bece9-759c-45b3-a39f-3f5cb00dc4e5 tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:07:41.340737 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c01bece9-759c-45b3-a39f-3f5cb00dc4e5 tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Acquiring lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 18 23:07:41.341088 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c01bece9-759c-45b3-a39f-3f5cb00dc4e5 tempest-DeleteServersTestJSON-956726688 tempest-DeleteServersTestJSON-956726688-project] Acquired lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 18 23:07:41.341821 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-f6756d7a-4c6a-4093-905c-ccb104ce78a3 None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:07:41.342320 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-f6756d7a-4c6a-4093-905c-ccb104ce78a3 None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:07:41.342826 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-f6756d7a-4c6a-4093-905c-ccb104ce78a3 None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:07:41.347865 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-f6756d7a-4c6a-4093-905c-ccb104ce78a3 None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:07:41.347865 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-f6756d7a-4c6a-4093-905c-ccb104ce78a3 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:07:41.347865 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-f6756d7a-4c6a-4093-905c-ccb104ce78a3 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:07:41.353665 n-d-853704-1 nova-conductor[120657]: ERROR nova.compute.rpcapi [None req-f6756d7a-4c6a-4093-905c-ccb104ce78a3 None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 18 23:07:41.354052 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-f6756d7a-4c6a-4093-905c-ccb104ce78a3 None None] Releasing lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 18 23:07:41.355007 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server [None req-f6756d7a-4c6a-4093-905c-ccb104ce78a3 None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 23:07:41.355007 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 23:07:41.355007 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 18 23:07:41.355007 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 18 23:07:41.355007 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 18 23:07:41.355007 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 23:07:41.355007 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 18 23:07:41.355007 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 23:07:41.355007 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 23:07:41.355007 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 18 23:07:41.355007 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 18 23:07:41.355007 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 18 23:07:41.355007 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 18 23:07:41.355007 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 18 23:07:41.355007 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 18 23:07:41.355007 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 18 23:07:41.355007 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 18 23:07:41.355007 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 18 23:07:41.355007 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 18 23:07:41.355007 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 18 23:07:41.355007 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 18 23:07:41.355007 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 18 23:07:41.355007 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 18 23:07:41.355007 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 23:07:41.355007 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 23:07:51.260383 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=9,nova_api:UPDATE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:07:51.261104 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=15,nova_api:DELETE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:07:51.350451 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:07:51.351264 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell1:SELECT=11,nova_cell1:INSERT=17 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:07:53.314944 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:07:53.345539 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=12,nova_cell0:UPDATE=3 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:08:33.309091 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:08:33.339866 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:09:03.314892 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=5,nova_cell0:UPDATE=3 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:09:03.346545 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=5,nova_cell0:UPDATE=3 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:09:33.355997 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=6,nova_cell0:UPDATE=3 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:09:43.319868 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:09:53.369531 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=4,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:10:13.325847 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=5,nova_cell0:UPDATE=3 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:10:13.366650 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=4,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:10:15.327870 n-d-853704-1 nova-conductor[120657]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 18 23:10:22.720063 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-d5216af5-933a-436f-a166-25e30c1d09ab tempest-ServerShowV254Test-1965110206 tempest-ServerShowV254Test-1965110206-project] Took 0.80 seconds to select destinations for 1 instance(s). {{(pid=120658) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 18 23:10:22.745198 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-d5216af5-933a-436f-a166-25e30c1d09ab tempest-ServerShowV254Test-1965110206 tempest-ServerShowV254Test-1965110206-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:10:22.745657 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-d5216af5-933a-436f-a166-25e30c1d09ab tempest-ServerShowV254Test-1965110206 tempest-ServerShowV254Test-1965110206-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:10:22.746061 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-d5216af5-933a-436f-a166-25e30c1d09ab tempest-ServerShowV254Test-1965110206 tempest-ServerShowV254Test-1965110206-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:10:22.918748 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-d5216af5-933a-436f-a166-25e30c1d09ab tempest-ServerShowV254Test-1965110206 tempest-ServerShowV254Test-1965110206-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:10:22.919184 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-d5216af5-933a-436f-a166-25e30c1d09ab tempest-ServerShowV254Test-1965110206 tempest-ServerShowV254Test-1965110206-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:10:22.919578 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-d5216af5-933a-436f-a166-25e30c1d09ab tempest-ServerShowV254Test-1965110206 tempest-ServerShowV254Test-1965110206-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:10:22.920275 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-d5216af5-933a-436f-a166-25e30c1d09ab tempest-ServerShowV254Test-1965110206 tempest-ServerShowV254Test-1965110206-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:10:22.920697 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-d5216af5-933a-436f-a166-25e30c1d09ab tempest-ServerShowV254Test-1965110206 tempest-ServerShowV254Test-1965110206-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:10:22.921113 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-d5216af5-933a-436f-a166-25e30c1d09ab tempest-ServerShowV254Test-1965110206 tempest-ServerShowV254Test-1965110206-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:10:22.936611 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-d5216af5-933a-436f-a166-25e30c1d09ab tempest-ServerShowV254Test-1965110206 tempest-ServerShowV254Test-1965110206-project] Getting quotas for project 54976dafb6c74e96884b1eaa9ffc34e0. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 18 23:10:22.941190 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-d5216af5-933a-436f-a166-25e30c1d09ab tempest-ServerShowV254Test-1965110206 tempest-ServerShowV254Test-1965110206-project] Getting quotas for user 2a7f897efc2c4540bf43730637b5b69a and project 54976dafb6c74e96884b1eaa9ffc34e0. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 18 23:10:22.952524 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-d5216af5-933a-436f-a166-25e30c1d09ab tempest-ServerShowV254Test-1965110206 tempest-ServerShowV254Test-1965110206-project] [instance: 2cf3c73f-6576-4c79-8510-48e6dbaad38e] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120658) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 18 23:10:22.953922 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-d5216af5-933a-436f-a166-25e30c1d09ab tempest-ServerShowV254Test-1965110206 tempest-ServerShowV254Test-1965110206-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:10:22.953922 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-d5216af5-933a-436f-a166-25e30c1d09ab tempest-ServerShowV254Test-1965110206 tempest-ServerShowV254Test-1965110206-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:10:22.954188 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-d5216af5-933a-436f-a166-25e30c1d09ab tempest-ServerShowV254Test-1965110206 tempest-ServerShowV254Test-1965110206-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:10:23.021867 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-d5216af5-933a-436f-a166-25e30c1d09ab tempest-ServerShowV254Test-1965110206 tempest-ServerShowV254Test-1965110206-project] [instance: 2cf3c73f-6576-4c79-8510-48e6dbaad38e] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='87b0fec9-a948-41f9-a3c0-f0a6a5a8f1e7',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120658) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 18 23:10:23.023211 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-d5216af5-933a-436f-a166-25e30c1d09ab tempest-ServerShowV254Test-1965110206 tempest-ServerShowV254Test-1965110206-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:10:23.024548 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-d5216af5-933a-436f-a166-25e30c1d09ab tempest-ServerShowV254Test-1965110206 tempest-ServerShowV254Test-1965110206-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:10:23.025009 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-d5216af5-933a-436f-a166-25e30c1d09ab tempest-ServerShowV254Test-1965110206 tempest-ServerShowV254Test-1965110206-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:10:23.314330 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-d5216af5-933a-436f-a166-25e30c1d09ab tempest-ServerShowV254Test-1965110206 tempest-ServerShowV254Test-1965110206-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:10:23.314703 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-d5216af5-933a-436f-a166-25e30c1d09ab tempest-ServerShowV254Test-1965110206 tempest-ServerShowV254Test-1965110206-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:10:23.315103 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-d5216af5-933a-436f-a166-25e30c1d09ab tempest-ServerShowV254Test-1965110206 tempest-ServerShowV254Test-1965110206-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:10:23.315553 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-d5216af5-933a-436f-a166-25e30c1d09ab tempest-ServerShowV254Test-1965110206 tempest-ServerShowV254Test-1965110206-project] Acquiring lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 18 23:10:23.315927 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-d5216af5-933a-436f-a166-25e30c1d09ab tempest-ServerShowV254Test-1965110206 tempest-ServerShowV254Test-1965110206-project] Acquired lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 18 23:10:23.316713 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-a9cc0db7-32d7-4fd5-afdc-366de609f591 None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:10:23.317128 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-a9cc0db7-32d7-4fd5-afdc-366de609f591 None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:10:23.317557 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-a9cc0db7-32d7-4fd5-afdc-366de609f591 None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:10:23.318209 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-a9cc0db7-32d7-4fd5-afdc-366de609f591 None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:10:23.318617 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-a9cc0db7-32d7-4fd5-afdc-366de609f591 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:10:23.319002 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-a9cc0db7-32d7-4fd5-afdc-366de609f591 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:10:23.333606 n-d-853704-1 nova-conductor[120658]: ERROR nova.compute.rpcapi [None req-a9cc0db7-32d7-4fd5-afdc-366de609f591 None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 18 23:10:23.334003 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-a9cc0db7-32d7-4fd5-afdc-366de609f591 None None] Releasing lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 18 23:10:23.334909 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server [None req-a9cc0db7-32d7-4fd5-afdc-366de609f591 None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 23:10:23.334909 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 23:10:23.334909 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 18 23:10:23.334909 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 18 23:10:23.334909 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 18 23:10:23.334909 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 23:10:23.334909 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 18 23:10:23.334909 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 23:10:23.334909 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 23:10:23.334909 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 18 23:10:23.334909 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 18 23:10:23.334909 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 18 23:10:23.334909 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 18 23:10:23.334909 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 18 23:10:23.334909 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 18 23:10:23.334909 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 18 23:10:23.334909 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 18 23:10:23.334909 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 18 23:10:23.334909 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 18 23:10:23.334909 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 18 23:10:23.334909 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 18 23:10:23.334909 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 18 23:10:23.334909 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 18 23:10:23.334909 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 23:10:23.334909 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 23:10:33.126295 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=14 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:10:33.236471 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=10,nova_api:UPDATE=1,nova_api:DELETE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:10:33.326871 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:10:33.329297 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell1:SELECT=11,nova_cell1:SAVEPOINT=1,nova_cell1:INSERT=18,nova_cell1:RELEASE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:10:53.320139 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=15,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:10:53.364756 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:11:33.331641 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:11:33.365692 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=3 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:11:39.687495 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-d9540080-6f9f-40a3-a189-b684283620b4 tempest-ServerAddressesNegativeTestJSON-1255180988 tempest-ServerAddressesNegativeTestJSON-1255180988-project] Took 0.68 seconds to select destinations for 1 instance(s). {{(pid=120657) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 18 23:11:39.713827 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-d9540080-6f9f-40a3-a189-b684283620b4 tempest-ServerAddressesNegativeTestJSON-1255180988 tempest-ServerAddressesNegativeTestJSON-1255180988-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:11:39.714485 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-d9540080-6f9f-40a3-a189-b684283620b4 tempest-ServerAddressesNegativeTestJSON-1255180988 tempest-ServerAddressesNegativeTestJSON-1255180988-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:11:39.715579 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-d9540080-6f9f-40a3-a189-b684283620b4 tempest-ServerAddressesNegativeTestJSON-1255180988 tempest-ServerAddressesNegativeTestJSON-1255180988-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:11:39.942451 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-d9540080-6f9f-40a3-a189-b684283620b4 tempest-ServerAddressesNegativeTestJSON-1255180988 tempest-ServerAddressesNegativeTestJSON-1255180988-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:11:39.944397 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-d9540080-6f9f-40a3-a189-b684283620b4 tempest-ServerAddressesNegativeTestJSON-1255180988 tempest-ServerAddressesNegativeTestJSON-1255180988-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:11:39.945333 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-d9540080-6f9f-40a3-a189-b684283620b4 tempest-ServerAddressesNegativeTestJSON-1255180988 tempest-ServerAddressesNegativeTestJSON-1255180988-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.002s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:11:39.946246 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-d9540080-6f9f-40a3-a189-b684283620b4 tempest-ServerAddressesNegativeTestJSON-1255180988 tempest-ServerAddressesNegativeTestJSON-1255180988-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:11:39.946896 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-d9540080-6f9f-40a3-a189-b684283620b4 tempest-ServerAddressesNegativeTestJSON-1255180988 tempest-ServerAddressesNegativeTestJSON-1255180988-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:11:39.947609 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-d9540080-6f9f-40a3-a189-b684283620b4 tempest-ServerAddressesNegativeTestJSON-1255180988 tempest-ServerAddressesNegativeTestJSON-1255180988-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:11:39.979491 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-d9540080-6f9f-40a3-a189-b684283620b4 tempest-ServerAddressesNegativeTestJSON-1255180988 tempest-ServerAddressesNegativeTestJSON-1255180988-project] Getting quotas for project b9c53889fe074193b4a68b0fda3aedca. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 18 23:11:39.984390 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-d9540080-6f9f-40a3-a189-b684283620b4 tempest-ServerAddressesNegativeTestJSON-1255180988 tempest-ServerAddressesNegativeTestJSON-1255180988-project] Getting quotas for user 6c74d32fbcae479f809f2b0740672578 and project b9c53889fe074193b4a68b0fda3aedca. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 18 23:11:39.996416 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-d9540080-6f9f-40a3-a189-b684283620b4 tempest-ServerAddressesNegativeTestJSON-1255180988 tempest-ServerAddressesNegativeTestJSON-1255180988-project] [instance: 433faab9-20f2-4ae4-b092-01c61da67d53] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120657) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 18 23:11:39.997556 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-d9540080-6f9f-40a3-a189-b684283620b4 tempest-ServerAddressesNegativeTestJSON-1255180988 tempest-ServerAddressesNegativeTestJSON-1255180988-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:11:39.999013 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-d9540080-6f9f-40a3-a189-b684283620b4 tempest-ServerAddressesNegativeTestJSON-1255180988 tempest-ServerAddressesNegativeTestJSON-1255180988-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:11:39.999472 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-d9540080-6f9f-40a3-a189-b684283620b4 tempest-ServerAddressesNegativeTestJSON-1255180988 tempest-ServerAddressesNegativeTestJSON-1255180988-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:11:40.053523 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-d9540080-6f9f-40a3-a189-b684283620b4 tempest-ServerAddressesNegativeTestJSON-1255180988 tempest-ServerAddressesNegativeTestJSON-1255180988-project] [instance: 433faab9-20f2-4ae4-b092-01c61da67d53] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='87b0fec9-a948-41f9-a3c0-f0a6a5a8f1e7',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120657) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 18 23:11:40.055160 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-d9540080-6f9f-40a3-a189-b684283620b4 tempest-ServerAddressesNegativeTestJSON-1255180988 tempest-ServerAddressesNegativeTestJSON-1255180988-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:11:40.055604 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-d9540080-6f9f-40a3-a189-b684283620b4 tempest-ServerAddressesNegativeTestJSON-1255180988 tempest-ServerAddressesNegativeTestJSON-1255180988-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:11:40.056163 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-d9540080-6f9f-40a3-a189-b684283620b4 tempest-ServerAddressesNegativeTestJSON-1255180988 tempest-ServerAddressesNegativeTestJSON-1255180988-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:11:40.329617 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-d9540080-6f9f-40a3-a189-b684283620b4 tempest-ServerAddressesNegativeTestJSON-1255180988 tempest-ServerAddressesNegativeTestJSON-1255180988-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:11:40.331131 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-d9540080-6f9f-40a3-a189-b684283620b4 tempest-ServerAddressesNegativeTestJSON-1255180988 tempest-ServerAddressesNegativeTestJSON-1255180988-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:11:40.331774 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-d9540080-6f9f-40a3-a189-b684283620b4 tempest-ServerAddressesNegativeTestJSON-1255180988 tempest-ServerAddressesNegativeTestJSON-1255180988-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:11:40.332451 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-d9540080-6f9f-40a3-a189-b684283620b4 tempest-ServerAddressesNegativeTestJSON-1255180988 tempest-ServerAddressesNegativeTestJSON-1255180988-project] Acquiring lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 18 23:11:40.333019 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-d9540080-6f9f-40a3-a189-b684283620b4 tempest-ServerAddressesNegativeTestJSON-1255180988 tempest-ServerAddressesNegativeTestJSON-1255180988-project] Acquired lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 18 23:11:40.334601 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-70a0d42a-d84b-4f4c-954c-0fc886649ab9 None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:11:40.335494 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-70a0d42a-d84b-4f4c-954c-0fc886649ab9 None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:11:40.336152 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-70a0d42a-d84b-4f4c-954c-0fc886649ab9 None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:11:40.337971 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-70a0d42a-d84b-4f4c-954c-0fc886649ab9 None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:11:40.338594 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-70a0d42a-d84b-4f4c-954c-0fc886649ab9 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.002s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:11:40.339625 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-70a0d42a-d84b-4f4c-954c-0fc886649ab9 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:11:40.355339 n-d-853704-1 nova-conductor[120657]: ERROR nova.compute.rpcapi [None req-70a0d42a-d84b-4f4c-954c-0fc886649ab9 None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 18 23:11:40.355339 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-70a0d42a-d84b-4f4c-954c-0fc886649ab9 None None] Releasing lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 18 23:11:40.355339 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server [None req-70a0d42a-d84b-4f4c-954c-0fc886649ab9 None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 23:11:40.355339 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 23:11:40.355339 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 18 23:11:40.355339 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 18 23:11:40.355339 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 18 23:11:40.355339 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 23:11:40.355339 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 18 23:11:40.355339 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 23:11:40.355339 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 23:11:40.355339 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 18 23:11:40.355339 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 18 23:11:40.355339 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 18 23:11:40.355339 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 18 23:11:40.355339 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 18 23:11:40.355339 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 18 23:11:40.355339 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 18 23:11:40.355339 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 18 23:11:40.355339 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 18 23:11:40.355339 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 18 23:11:40.355339 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 18 23:11:40.355339 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 18 23:11:40.355339 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 18 23:11:40.355339 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 18 23:11:40.355339 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 23:11:40.355339 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 23:11:45.703536 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-650b7fb3-42e2-476c-9b13-39dd8b1b4bcf tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Took 0.43 seconds to select destinations for 1 instance(s). {{(pid=120658) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 18 23:11:45.733026 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-650b7fb3-42e2-476c-9b13-39dd8b1b4bcf tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:11:45.733026 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-650b7fb3-42e2-476c-9b13-39dd8b1b4bcf tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:11:45.733026 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-650b7fb3-42e2-476c-9b13-39dd8b1b4bcf tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:11:45.877298 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-650b7fb3-42e2-476c-9b13-39dd8b1b4bcf tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:11:45.877740 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-650b7fb3-42e2-476c-9b13-39dd8b1b4bcf tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:11:45.878130 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-650b7fb3-42e2-476c-9b13-39dd8b1b4bcf tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:11:45.878786 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-650b7fb3-42e2-476c-9b13-39dd8b1b4bcf tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:11:45.879194 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-650b7fb3-42e2-476c-9b13-39dd8b1b4bcf tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:11:45.879575 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-650b7fb3-42e2-476c-9b13-39dd8b1b4bcf tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:11:45.896635 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-650b7fb3-42e2-476c-9b13-39dd8b1b4bcf tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Getting quotas for project 05baaf842a13444d8905683625e5853a. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 18 23:11:45.903924 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-650b7fb3-42e2-476c-9b13-39dd8b1b4bcf tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Getting quotas for user 4d73c623dd454555ac8d974cf7a60398 and project 05baaf842a13444d8905683625e5853a. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 18 23:11:45.915860 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-650b7fb3-42e2-476c-9b13-39dd8b1b4bcf tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] [instance: 159f52db-ffec-4ae7-8e32-3d6695936f6c] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120658) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 18 23:11:45.920348 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-650b7fb3-42e2-476c-9b13-39dd8b1b4bcf tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:11:45.920348 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-650b7fb3-42e2-476c-9b13-39dd8b1b4bcf tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:11:45.920348 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-650b7fb3-42e2-476c-9b13-39dd8b1b4bcf tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:11:45.991943 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-650b7fb3-42e2-476c-9b13-39dd8b1b4bcf tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] [instance: 159f52db-ffec-4ae7-8e32-3d6695936f6c] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='87b0fec9-a948-41f9-a3c0-f0a6a5a8f1e7',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120658) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 18 23:11:45.993163 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-650b7fb3-42e2-476c-9b13-39dd8b1b4bcf tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:11:45.995580 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-650b7fb3-42e2-476c-9b13-39dd8b1b4bcf tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:11:45.995580 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-650b7fb3-42e2-476c-9b13-39dd8b1b4bcf tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:11:46.260110 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-650b7fb3-42e2-476c-9b13-39dd8b1b4bcf tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:11:46.260579 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-650b7fb3-42e2-476c-9b13-39dd8b1b4bcf tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:11:46.260972 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-650b7fb3-42e2-476c-9b13-39dd8b1b4bcf tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:11:46.261419 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-650b7fb3-42e2-476c-9b13-39dd8b1b4bcf tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Acquiring lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 18 23:11:46.261781 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-650b7fb3-42e2-476c-9b13-39dd8b1b4bcf tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Acquired lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 18 23:11:46.262534 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-1fe9eee6-3898-4707-a0a8-a886825d1280 None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:11:46.263002 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-1fe9eee6-3898-4707-a0a8-a886825d1280 None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:11:46.263376 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-1fe9eee6-3898-4707-a0a8-a886825d1280 None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:11:46.264048 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-1fe9eee6-3898-4707-a0a8-a886825d1280 None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:11:46.264450 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-1fe9eee6-3898-4707-a0a8-a886825d1280 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:11:46.264818 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-1fe9eee6-3898-4707-a0a8-a886825d1280 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:11:46.273778 n-d-853704-1 nova-conductor[120658]: ERROR nova.compute.rpcapi [None req-1fe9eee6-3898-4707-a0a8-a886825d1280 None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 18 23:11:46.274162 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-1fe9eee6-3898-4707-a0a8-a886825d1280 None None] Releasing lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 18 23:11:46.275088 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server [None req-1fe9eee6-3898-4707-a0a8-a886825d1280 None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 23:11:46.275088 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 23:11:46.275088 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 18 23:11:46.275088 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 18 23:11:46.275088 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 18 23:11:46.275088 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 23:11:46.275088 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 18 23:11:46.275088 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 23:11:46.275088 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 23:11:46.275088 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 18 23:11:46.275088 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 18 23:11:46.275088 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 18 23:11:46.275088 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 18 23:11:46.275088 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 18 23:11:46.275088 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 18 23:11:46.275088 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 18 23:11:46.275088 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 18 23:11:46.275088 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 18 23:11:46.275088 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 18 23:11:46.275088 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 18 23:11:46.275088 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 18 23:11:46.275088 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 18 23:11:46.275088 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 18 23:11:46.275088 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 23:11:46.275088 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 23:11:50.156126 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=15 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:11:50.235058 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=9,nova_api:UPDATE=1,nova_api:DELETE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:11:50.353251 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:11:50.353949 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell1:SELECT=11,nova_cell1:SAVEPOINT=1,nova_cell1:INSERT=18,nova_cell1:RELEASE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:11:56.182551 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=9,nova_api:UPDATE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:11:56.183350 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=15,nova_api:DELETE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:11:56.271271 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:11:56.271814 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell1:SELECT=11,nova_cell1:INSERT=17 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:12:13.327955 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=14,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:12:13.355159 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=14,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:12:53.331649 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:12:53.363684 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:13:33.264155 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-3a0e89b1-028c-4719-acd2-e6995ac0af69 tempest-TestVolumeBootPattern-1034437477 tempest-TestVolumeBootPattern-1034437477-project] Took 0.74 seconds to select destinations for 1 instance(s). {{(pid=120657) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 18 23:13:33.288437 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-3a0e89b1-028c-4719-acd2-e6995ac0af69 tempest-TestVolumeBootPattern-1034437477 tempest-TestVolumeBootPattern-1034437477-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:13:33.288928 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-3a0e89b1-028c-4719-acd2-e6995ac0af69 tempest-TestVolumeBootPattern-1034437477 tempest-TestVolumeBootPattern-1034437477-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:13:33.293000 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-3a0e89b1-028c-4719-acd2-e6995ac0af69 tempest-TestVolumeBootPattern-1034437477 tempest-TestVolumeBootPattern-1034437477-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:13:33.332406 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:13:33.353457 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:13:33.487203 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-3a0e89b1-028c-4719-acd2-e6995ac0af69 tempest-TestVolumeBootPattern-1034437477 tempest-TestVolumeBootPattern-1034437477-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:13:33.487650 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-3a0e89b1-028c-4719-acd2-e6995ac0af69 tempest-TestVolumeBootPattern-1034437477 tempest-TestVolumeBootPattern-1034437477-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:13:33.488547 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-3a0e89b1-028c-4719-acd2-e6995ac0af69 tempest-TestVolumeBootPattern-1034437477 tempest-TestVolumeBootPattern-1034437477-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:13:33.488755 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-3a0e89b1-028c-4719-acd2-e6995ac0af69 tempest-TestVolumeBootPattern-1034437477 tempest-TestVolumeBootPattern-1034437477-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:13:33.489168 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-3a0e89b1-028c-4719-acd2-e6995ac0af69 tempest-TestVolumeBootPattern-1034437477 tempest-TestVolumeBootPattern-1034437477-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:13:33.489693 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-3a0e89b1-028c-4719-acd2-e6995ac0af69 tempest-TestVolumeBootPattern-1034437477 tempest-TestVolumeBootPattern-1034437477-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:13:33.508719 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-3a0e89b1-028c-4719-acd2-e6995ac0af69 tempest-TestVolumeBootPattern-1034437477 tempest-TestVolumeBootPattern-1034437477-project] Getting quotas for project dfab7231c21e4b1f82cb2fbf1c4e0986. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 18 23:13:33.513968 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-3a0e89b1-028c-4719-acd2-e6995ac0af69 tempest-TestVolumeBootPattern-1034437477 tempest-TestVolumeBootPattern-1034437477-project] Getting quotas for user 480cd2375ac24f8a89b9730fcaa6965e and project dfab7231c21e4b1f82cb2fbf1c4e0986. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 18 23:13:33.524969 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-3a0e89b1-028c-4719-acd2-e6995ac0af69 tempest-TestVolumeBootPattern-1034437477 tempest-TestVolumeBootPattern-1034437477-project] [instance: 48970a53-48e1-4c80-9c10-f30a6e46ef27] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120657) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 18 23:13:33.526194 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-3a0e89b1-028c-4719-acd2-e6995ac0af69 tempest-TestVolumeBootPattern-1034437477 tempest-TestVolumeBootPattern-1034437477-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:13:33.526648 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-3a0e89b1-028c-4719-acd2-e6995ac0af69 tempest-TestVolumeBootPattern-1034437477 tempest-TestVolumeBootPattern-1034437477-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:13:33.527038 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-3a0e89b1-028c-4719-acd2-e6995ac0af69 tempest-TestVolumeBootPattern-1034437477 tempest-TestVolumeBootPattern-1034437477-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:13:33.634697 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-3a0e89b1-028c-4719-acd2-e6995ac0af69 tempest-TestVolumeBootPattern-1034437477 tempest-TestVolumeBootPattern-1034437477-project] [instance: 48970a53-48e1-4c80-9c10-f30a6e46ef27] block_device_mapping [BlockDeviceMapping(attachment_id=1679501e-ece2-40b9-9f89-0130e7f4ca2e,boot_index=0,connection_info=None,created_at=,delete_on_termination=False,deleted=,deleted_at=,destination_type='volume',device_name=None,device_type=None,disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id=None,instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='volume',tag=None,updated_at=,uuid=,volume_id='2d5c9435-9f20-4334-b69f-6b39cc9590af',volume_size=1,volume_type=None)] {{(pid=120657) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 18 23:13:33.636595 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-3a0e89b1-028c-4719-acd2-e6995ac0af69 tempest-TestVolumeBootPattern-1034437477 tempest-TestVolumeBootPattern-1034437477-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:13:33.637062 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-3a0e89b1-028c-4719-acd2-e6995ac0af69 tempest-TestVolumeBootPattern-1034437477 tempest-TestVolumeBootPattern-1034437477-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:13:33.637519 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-3a0e89b1-028c-4719-acd2-e6995ac0af69 tempest-TestVolumeBootPattern-1034437477 tempest-TestVolumeBootPattern-1034437477-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:13:33.858534 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-3a0e89b1-028c-4719-acd2-e6995ac0af69 tempest-TestVolumeBootPattern-1034437477 tempest-TestVolumeBootPattern-1034437477-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:13:33.859044 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-3a0e89b1-028c-4719-acd2-e6995ac0af69 tempest-TestVolumeBootPattern-1034437477 tempest-TestVolumeBootPattern-1034437477-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:13:33.859474 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-3a0e89b1-028c-4719-acd2-e6995ac0af69 tempest-TestVolumeBootPattern-1034437477 tempest-TestVolumeBootPattern-1034437477-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:13:33.860000 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-3a0e89b1-028c-4719-acd2-e6995ac0af69 tempest-TestVolumeBootPattern-1034437477 tempest-TestVolumeBootPattern-1034437477-project] Acquiring lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 18 23:13:33.860580 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-3a0e89b1-028c-4719-acd2-e6995ac0af69 tempest-TestVolumeBootPattern-1034437477 tempest-TestVolumeBootPattern-1034437477-project] Acquired lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 18 23:13:33.861790 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-ec1d6f26-5b3d-4778-8370-28baeac5a303 None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:13:33.862212 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-ec1d6f26-5b3d-4778-8370-28baeac5a303 None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:13:33.862885 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-ec1d6f26-5b3d-4778-8370-28baeac5a303 None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:13:33.863602 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-ec1d6f26-5b3d-4778-8370-28baeac5a303 None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:13:33.866118 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-ec1d6f26-5b3d-4778-8370-28baeac5a303 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:13:33.866118 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-ec1d6f26-5b3d-4778-8370-28baeac5a303 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:13:33.878042 n-d-853704-1 nova-conductor[120657]: ERROR nova.compute.rpcapi [None req-ec1d6f26-5b3d-4778-8370-28baeac5a303 None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 18 23:13:33.878879 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-ec1d6f26-5b3d-4778-8370-28baeac5a303 None None] Releasing lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 18 23:13:33.880116 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server [None req-ec1d6f26-5b3d-4778-8370-28baeac5a303 None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 23:13:33.880116 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 23:13:33.880116 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 18 23:13:33.880116 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 18 23:13:33.880116 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 18 23:13:33.880116 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 23:13:33.880116 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 18 23:13:33.880116 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 23:13:33.880116 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 23:13:33.880116 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 18 23:13:33.880116 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 18 23:13:33.880116 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 18 23:13:33.880116 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 18 23:13:33.880116 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 18 23:13:33.880116 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 18 23:13:33.880116 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 18 23:13:33.880116 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 18 23:13:33.880116 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 18 23:13:33.880116 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 18 23:13:33.880116 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 18 23:13:33.880116 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 18 23:13:33.880116 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 18 23:13:33.880116 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 18 23:13:33.880116 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 23:13:33.880116 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 23:13:35.337809 n-d-853704-1 nova-conductor[120657]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 18 23:13:43.736289 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=11,nova_api:UPDATE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:13:43.797163 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=13,nova_api:DELETE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:13:43.877599 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:13:43.878522 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell1:SELECT=11,nova_cell1:SAVEPOINT=1,nova_cell1:INSERT=12,nova_cell1:RELEASE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:13:49.733823 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-07697604-a27f-480b-8341-5a4ce957964e tempest-ServerShowV257Test-157925070 tempest-ServerShowV257Test-157925070-project] Took 1.53 seconds to select destinations for 1 instance(s). {{(pid=120658) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 18 23:13:49.762078 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-07697604-a27f-480b-8341-5a4ce957964e tempest-ServerShowV257Test-157925070 tempest-ServerShowV257Test-157925070-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:13:49.763081 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-07697604-a27f-480b-8341-5a4ce957964e tempest-ServerShowV257Test-157925070 tempest-ServerShowV257Test-157925070-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:13:49.763802 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-07697604-a27f-480b-8341-5a4ce957964e tempest-ServerShowV257Test-157925070 tempest-ServerShowV257Test-157925070-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:13:49.921223 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-07697604-a27f-480b-8341-5a4ce957964e tempest-ServerShowV257Test-157925070 tempest-ServerShowV257Test-157925070-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:13:49.921670 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-07697604-a27f-480b-8341-5a4ce957964e tempest-ServerShowV257Test-157925070 tempest-ServerShowV257Test-157925070-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:13:49.922061 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-07697604-a27f-480b-8341-5a4ce957964e tempest-ServerShowV257Test-157925070 tempest-ServerShowV257Test-157925070-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:13:49.922728 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-07697604-a27f-480b-8341-5a4ce957964e tempest-ServerShowV257Test-157925070 tempest-ServerShowV257Test-157925070-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:13:49.923136 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-07697604-a27f-480b-8341-5a4ce957964e tempest-ServerShowV257Test-157925070 tempest-ServerShowV257Test-157925070-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:13:49.923525 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-07697604-a27f-480b-8341-5a4ce957964e tempest-ServerShowV257Test-157925070 tempest-ServerShowV257Test-157925070-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:13:49.941887 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-07697604-a27f-480b-8341-5a4ce957964e tempest-ServerShowV257Test-157925070 tempest-ServerShowV257Test-157925070-project] Getting quotas for project 15ab161601164754a45019e410da30f8. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 18 23:13:49.946441 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-07697604-a27f-480b-8341-5a4ce957964e tempest-ServerShowV257Test-157925070 tempest-ServerShowV257Test-157925070-project] Getting quotas for user 3e0a2d92ceae47699d897ffa980adb70 and project 15ab161601164754a45019e410da30f8. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 18 23:13:49.957863 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-07697604-a27f-480b-8341-5a4ce957964e tempest-ServerShowV257Test-157925070 tempest-ServerShowV257Test-157925070-project] [instance: 24a05b55-c272-4a39-b8c1-598d6f004492] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120658) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 18 23:13:49.958654 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-07697604-a27f-480b-8341-5a4ce957964e tempest-ServerShowV257Test-157925070 tempest-ServerShowV257Test-157925070-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:13:49.959091 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-07697604-a27f-480b-8341-5a4ce957964e tempest-ServerShowV257Test-157925070 tempest-ServerShowV257Test-157925070-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:13:49.959480 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-07697604-a27f-480b-8341-5a4ce957964e tempest-ServerShowV257Test-157925070 tempest-ServerShowV257Test-157925070-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:13:50.023643 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-07697604-a27f-480b-8341-5a4ce957964e tempest-ServerShowV257Test-157925070 tempest-ServerShowV257Test-157925070-project] [instance: 24a05b55-c272-4a39-b8c1-598d6f004492] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='87b0fec9-a948-41f9-a3c0-f0a6a5a8f1e7',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120658) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 18 23:13:50.024804 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-07697604-a27f-480b-8341-5a4ce957964e tempest-ServerShowV257Test-157925070 tempest-ServerShowV257Test-157925070-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:13:50.025289 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-07697604-a27f-480b-8341-5a4ce957964e tempest-ServerShowV257Test-157925070 tempest-ServerShowV257Test-157925070-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:13:50.025678 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-07697604-a27f-480b-8341-5a4ce957964e tempest-ServerShowV257Test-157925070 tempest-ServerShowV257Test-157925070-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:13:50.252598 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-07697604-a27f-480b-8341-5a4ce957964e tempest-ServerShowV257Test-157925070 tempest-ServerShowV257Test-157925070-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:13:50.252949 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-07697604-a27f-480b-8341-5a4ce957964e tempest-ServerShowV257Test-157925070 tempest-ServerShowV257Test-157925070-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:13:50.253339 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-07697604-a27f-480b-8341-5a4ce957964e tempest-ServerShowV257Test-157925070 tempest-ServerShowV257Test-157925070-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:13:50.253810 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-07697604-a27f-480b-8341-5a4ce957964e tempest-ServerShowV257Test-157925070 tempest-ServerShowV257Test-157925070-project] Acquiring lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 18 23:13:50.254154 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-07697604-a27f-480b-8341-5a4ce957964e tempest-ServerShowV257Test-157925070 tempest-ServerShowV257Test-157925070-project] Acquired lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 18 23:13:50.254873 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-cdc2d977-9bc9-4a0f-8e1d-2bef248b4402 None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:13:50.255275 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-cdc2d977-9bc9-4a0f-8e1d-2bef248b4402 None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:13:50.256004 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-cdc2d977-9bc9-4a0f-8e1d-2bef248b4402 None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:13:50.256647 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-cdc2d977-9bc9-4a0f-8e1d-2bef248b4402 None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:13:50.257048 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-cdc2d977-9bc9-4a0f-8e1d-2bef248b4402 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:13:50.257425 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-cdc2d977-9bc9-4a0f-8e1d-2bef248b4402 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:13:50.266339 n-d-853704-1 nova-conductor[120658]: ERROR nova.compute.rpcapi [None req-cdc2d977-9bc9-4a0f-8e1d-2bef248b4402 None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 18 23:13:50.270255 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-cdc2d977-9bc9-4a0f-8e1d-2bef248b4402 None None] Releasing lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 18 23:13:50.270255 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server [None req-cdc2d977-9bc9-4a0f-8e1d-2bef248b4402 None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 23:13:50.270255 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 23:13:50.270255 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 18 23:13:50.270255 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 18 23:13:50.270255 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 18 23:13:50.270255 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 23:13:50.270255 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 18 23:13:50.270255 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 23:13:50.270255 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 23:13:50.270255 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 18 23:13:50.270255 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 18 23:13:50.270255 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 18 23:13:50.270255 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 18 23:13:50.270255 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 18 23:13:50.270255 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 18 23:13:50.270255 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 18 23:13:50.270255 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 18 23:13:50.270255 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 18 23:13:50.270255 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 18 23:13:50.270255 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 18 23:13:50.270255 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 18 23:13:50.270255 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 18 23:13:50.270255 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 18 23:13:50.270255 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 23:13:50.270255 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 23:14:00.102560 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=11 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:14:00.477661 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=13,nova_api:UPDATE=1,nova_api:DELETE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:14:00.477661 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:14:00.477661 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell1:SELECT=11,nova_cell1:SAVEPOINT=1,nova_cell1:INSERT=18,nova_cell1:RELEASE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:14:13.331949 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=14,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:14:13.353618 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=14,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:14:35.058524 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-c026618b-144d-45b0-ad1c-b57f296b476e tempest-TestServerMultinode-1026161691 tempest-TestServerMultinode-1026161691-project-admin] Took 0.76 seconds to select destinations for 1 instance(s). {{(pid=120657) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 18 23:14:35.080021 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c026618b-144d-45b0-ad1c-b57f296b476e tempest-TestServerMultinode-1026161691 tempest-TestServerMultinode-1026161691-project-admin] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:14:35.080482 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c026618b-144d-45b0-ad1c-b57f296b476e tempest-TestServerMultinode-1026161691 tempest-TestServerMultinode-1026161691-project-admin] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:14:35.080960 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c026618b-144d-45b0-ad1c-b57f296b476e tempest-TestServerMultinode-1026161691 tempest-TestServerMultinode-1026161691-project-admin] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:14:35.304371 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c026618b-144d-45b0-ad1c-b57f296b476e tempest-TestServerMultinode-1026161691 tempest-TestServerMultinode-1026161691-project-admin] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:14:35.304987 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c026618b-144d-45b0-ad1c-b57f296b476e tempest-TestServerMultinode-1026161691 tempest-TestServerMultinode-1026161691-project-admin] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:14:35.305589 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c026618b-144d-45b0-ad1c-b57f296b476e tempest-TestServerMultinode-1026161691 tempest-TestServerMultinode-1026161691-project-admin] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:14:35.306269 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c026618b-144d-45b0-ad1c-b57f296b476e tempest-TestServerMultinode-1026161691 tempest-TestServerMultinode-1026161691-project-admin] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:14:35.306685 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c026618b-144d-45b0-ad1c-b57f296b476e tempest-TestServerMultinode-1026161691 tempest-TestServerMultinode-1026161691-project-admin] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:14:35.307087 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c026618b-144d-45b0-ad1c-b57f296b476e tempest-TestServerMultinode-1026161691 tempest-TestServerMultinode-1026161691-project-admin] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:14:35.323342 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-c026618b-144d-45b0-ad1c-b57f296b476e tempest-TestServerMultinode-1026161691 tempest-TestServerMultinode-1026161691-project-admin] Getting quotas for project 7c18c7fcfb064460b34f61919104bb8d. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 18 23:14:35.327858 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-c026618b-144d-45b0-ad1c-b57f296b476e tempest-TestServerMultinode-1026161691 tempest-TestServerMultinode-1026161691-project-admin] Getting quotas for user 17e0386a200449048ac75256aa170bd2 and project 7c18c7fcfb064460b34f61919104bb8d. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 18 23:14:35.339709 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-c026618b-144d-45b0-ad1c-b57f296b476e tempest-TestServerMultinode-1026161691 tempest-TestServerMultinode-1026161691-project-admin] [instance: f20b76b8-7673-440b-b8cb-a2f7eed9a1ab] Selected host: n-h1-853704-1; Selected node: n-h1-853704-1; Alternates: [] {{(pid=120657) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 18 23:14:35.339709 n-d-853704-1 nova-conductor[120657]: DEBUG nova.scheduler.utils [None req-c026618b-144d-45b0-ad1c-b57f296b476e tempest-TestServerMultinode-1026161691 tempest-TestServerMultinode-1026161691-project-admin] Re-scheduling is disabled due to forcing a host (['n-h1-853704-1']) and/or node ([]) {{(pid=120657) populate_retry /opt/stack/nova/nova/scheduler/utils.py:976}} Aug 18 23:14:35.341307 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c026618b-144d-45b0-ad1c-b57f296b476e tempest-TestServerMultinode-1026161691 tempest-TestServerMultinode-1026161691-project-admin] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:14:35.341535 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c026618b-144d-45b0-ad1c-b57f296b476e tempest-TestServerMultinode-1026161691 tempest-TestServerMultinode-1026161691-project-admin] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:14:35.342015 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c026618b-144d-45b0-ad1c-b57f296b476e tempest-TestServerMultinode-1026161691 tempest-TestServerMultinode-1026161691-project-admin] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:14:35.413311 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-c026618b-144d-45b0-ad1c-b57f296b476e tempest-TestServerMultinode-1026161691 tempest-TestServerMultinode-1026161691-project-admin] [instance: f20b76b8-7673-440b-b8cb-a2f7eed9a1ab] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='87b0fec9-a948-41f9-a3c0-f0a6a5a8f1e7',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120657) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 18 23:14:35.414448 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c026618b-144d-45b0-ad1c-b57f296b476e tempest-TestServerMultinode-1026161691 tempest-TestServerMultinode-1026161691-project-admin] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:14:35.414892 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c026618b-144d-45b0-ad1c-b57f296b476e tempest-TestServerMultinode-1026161691 tempest-TestServerMultinode-1026161691-project-admin] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:14:35.415289 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c026618b-144d-45b0-ad1c-b57f296b476e tempest-TestServerMultinode-1026161691 tempest-TestServerMultinode-1026161691-project-admin] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:14:35.648761 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c026618b-144d-45b0-ad1c-b57f296b476e tempest-TestServerMultinode-1026161691 tempest-TestServerMultinode-1026161691-project-admin] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:14:35.649215 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c026618b-144d-45b0-ad1c-b57f296b476e tempest-TestServerMultinode-1026161691 tempest-TestServerMultinode-1026161691-project-admin] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:14:35.649748 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c026618b-144d-45b0-ad1c-b57f296b476e tempest-TestServerMultinode-1026161691 tempest-TestServerMultinode-1026161691-project-admin] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:14:35.650463 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c026618b-144d-45b0-ad1c-b57f296b476e tempest-TestServerMultinode-1026161691 tempest-TestServerMultinode-1026161691-project-admin] Acquiring lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 18 23:14:35.650983 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c026618b-144d-45b0-ad1c-b57f296b476e tempest-TestServerMultinode-1026161691 tempest-TestServerMultinode-1026161691-project-admin] Acquired lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 18 23:14:35.651756 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-8af79e4e-8dfa-4a3d-99d5-4f6000dcf7a1 None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:14:35.655371 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-8af79e4e-8dfa-4a3d-99d5-4f6000dcf7a1 None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:14:35.655371 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-8af79e4e-8dfa-4a3d-99d5-4f6000dcf7a1 None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:14:35.655371 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-8af79e4e-8dfa-4a3d-99d5-4f6000dcf7a1 None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:14:35.655371 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-8af79e4e-8dfa-4a3d-99d5-4f6000dcf7a1 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:14:35.655371 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-8af79e4e-8dfa-4a3d-99d5-4f6000dcf7a1 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:14:35.662727 n-d-853704-1 nova-conductor[120657]: ERROR nova.compute.rpcapi [None req-8af79e4e-8dfa-4a3d-99d5-4f6000dcf7a1 None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 18 23:14:35.663992 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-8af79e4e-8dfa-4a3d-99d5-4f6000dcf7a1 None None] Releasing lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 18 23:14:35.665742 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server [None req-8af79e4e-8dfa-4a3d-99d5-4f6000dcf7a1 None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 23:14:35.665742 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 23:14:35.665742 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 18 23:14:35.665742 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 18 23:14:35.665742 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 18 23:14:35.665742 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 23:14:35.665742 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 18 23:14:35.665742 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 23:14:35.665742 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 23:14:35.665742 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 18 23:14:35.665742 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 18 23:14:35.665742 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 18 23:14:35.665742 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 18 23:14:35.665742 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 18 23:14:35.665742 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 18 23:14:35.665742 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 18 23:14:35.665742 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 18 23:14:35.665742 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 18 23:14:35.665742 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 18 23:14:35.665742 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 18 23:14:35.665742 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 18 23:14:35.665742 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 18 23:14:35.665742 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 18 23:14:35.665742 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 23:14:35.665742 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 23:14:45.507008 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=16 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:14:45.595106 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=8,nova_api:UPDATE=1,nova_api:DELETE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:14:45.660926 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:14:45.661703 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell1:SELECT=11,nova_cell1:SAVEPOINT=1,nova_cell1:INSERT=18,nova_cell1:RELEASE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:14:53.328747 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:14:53.349336 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=14,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:15:33.323191 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:15:33.345351 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:15:53.328187 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=3,nova_cell0:UPDATE=2 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:15:53.349303 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=3,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:16:13.328997 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=4,nova_cell0:UPDATE=2 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:16:13.360828 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=4,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:16:40.946584 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-b094b326-5029-4a79-a651-c35e73ff58f6 tempest-ServersNegativeTestMultiTenantJSON-1849416925 tempest-ServersNegativeTestMultiTenantJSON-1849416925-project] Took 1.63 seconds to select destinations for 1 instance(s). {{(pid=120658) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 18 23:16:40.967559 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-b094b326-5029-4a79-a651-c35e73ff58f6 tempest-ServersNegativeTestMultiTenantJSON-1849416925 tempest-ServersNegativeTestMultiTenantJSON-1849416925-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:16:40.967934 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-b094b326-5029-4a79-a651-c35e73ff58f6 tempest-ServersNegativeTestMultiTenantJSON-1849416925 tempest-ServersNegativeTestMultiTenantJSON-1849416925-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:16:40.968322 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-b094b326-5029-4a79-a651-c35e73ff58f6 tempest-ServersNegativeTestMultiTenantJSON-1849416925 tempest-ServersNegativeTestMultiTenantJSON-1849416925-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:16:41.136257 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-b094b326-5029-4a79-a651-c35e73ff58f6 tempest-ServersNegativeTestMultiTenantJSON-1849416925 tempest-ServersNegativeTestMultiTenantJSON-1849416925-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:16:41.136587 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-b094b326-5029-4a79-a651-c35e73ff58f6 tempest-ServersNegativeTestMultiTenantJSON-1849416925 tempest-ServersNegativeTestMultiTenantJSON-1849416925-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:16:41.136990 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-b094b326-5029-4a79-a651-c35e73ff58f6 tempest-ServersNegativeTestMultiTenantJSON-1849416925 tempest-ServersNegativeTestMultiTenantJSON-1849416925-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:16:41.137703 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-b094b326-5029-4a79-a651-c35e73ff58f6 tempest-ServersNegativeTestMultiTenantJSON-1849416925 tempest-ServersNegativeTestMultiTenantJSON-1849416925-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:16:41.138727 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-b094b326-5029-4a79-a651-c35e73ff58f6 tempest-ServersNegativeTestMultiTenantJSON-1849416925 tempest-ServersNegativeTestMultiTenantJSON-1849416925-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:16:41.139151 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-b094b326-5029-4a79-a651-c35e73ff58f6 tempest-ServersNegativeTestMultiTenantJSON-1849416925 tempest-ServersNegativeTestMultiTenantJSON-1849416925-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:16:41.157592 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-b094b326-5029-4a79-a651-c35e73ff58f6 tempest-ServersNegativeTestMultiTenantJSON-1849416925 tempest-ServersNegativeTestMultiTenantJSON-1849416925-project] Getting quotas for project 8803a7f621ee41568ea90693c525931d. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 18 23:16:41.161045 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-b094b326-5029-4a79-a651-c35e73ff58f6 tempest-ServersNegativeTestMultiTenantJSON-1849416925 tempest-ServersNegativeTestMultiTenantJSON-1849416925-project] Getting quotas for user 730a7431670241f3920da6b45bc9d403 and project 8803a7f621ee41568ea90693c525931d. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 18 23:16:41.173436 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-b094b326-5029-4a79-a651-c35e73ff58f6 tempest-ServersNegativeTestMultiTenantJSON-1849416925 tempest-ServersNegativeTestMultiTenantJSON-1849416925-project] [instance: bcdfb81e-8f9e-482a-9134-02089a83709d] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120658) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 18 23:16:41.174521 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-b094b326-5029-4a79-a651-c35e73ff58f6 tempest-ServersNegativeTestMultiTenantJSON-1849416925 tempest-ServersNegativeTestMultiTenantJSON-1849416925-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:16:41.174962 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-b094b326-5029-4a79-a651-c35e73ff58f6 tempest-ServersNegativeTestMultiTenantJSON-1849416925 tempest-ServersNegativeTestMultiTenantJSON-1849416925-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:16:41.175357 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-b094b326-5029-4a79-a651-c35e73ff58f6 tempest-ServersNegativeTestMultiTenantJSON-1849416925 tempest-ServersNegativeTestMultiTenantJSON-1849416925-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:16:41.238644 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-b094b326-5029-4a79-a651-c35e73ff58f6 tempest-ServersNegativeTestMultiTenantJSON-1849416925 tempest-ServersNegativeTestMultiTenantJSON-1849416925-project] [instance: bcdfb81e-8f9e-482a-9134-02089a83709d] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='87b0fec9-a948-41f9-a3c0-f0a6a5a8f1e7',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120658) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 18 23:16:41.239867 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-b094b326-5029-4a79-a651-c35e73ff58f6 tempest-ServersNegativeTestMultiTenantJSON-1849416925 tempest-ServersNegativeTestMultiTenantJSON-1849416925-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:16:41.240321 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-b094b326-5029-4a79-a651-c35e73ff58f6 tempest-ServersNegativeTestMultiTenantJSON-1849416925 tempest-ServersNegativeTestMultiTenantJSON-1849416925-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:16:41.240769 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-b094b326-5029-4a79-a651-c35e73ff58f6 tempest-ServersNegativeTestMultiTenantJSON-1849416925 tempest-ServersNegativeTestMultiTenantJSON-1849416925-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:16:41.405550 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-b094b326-5029-4a79-a651-c35e73ff58f6 tempest-ServersNegativeTestMultiTenantJSON-1849416925 tempest-ServersNegativeTestMultiTenantJSON-1849416925-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:16:41.406122 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-b094b326-5029-4a79-a651-c35e73ff58f6 tempest-ServersNegativeTestMultiTenantJSON-1849416925 tempest-ServersNegativeTestMultiTenantJSON-1849416925-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:16:41.406644 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-b094b326-5029-4a79-a651-c35e73ff58f6 tempest-ServersNegativeTestMultiTenantJSON-1849416925 tempest-ServersNegativeTestMultiTenantJSON-1849416925-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:16:41.407279 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-b094b326-5029-4a79-a651-c35e73ff58f6 tempest-ServersNegativeTestMultiTenantJSON-1849416925 tempest-ServersNegativeTestMultiTenantJSON-1849416925-project] Acquiring lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 18 23:16:41.410367 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-b094b326-5029-4a79-a651-c35e73ff58f6 tempest-ServersNegativeTestMultiTenantJSON-1849416925 tempest-ServersNegativeTestMultiTenantJSON-1849416925-project] Acquired lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 18 23:16:41.410367 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-c18ec6d5-21d7-4adc-9c29-0e59f2ffc438 None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:16:41.410367 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-c18ec6d5-21d7-4adc-9c29-0e59f2ffc438 None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:16:41.410367 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-c18ec6d5-21d7-4adc-9c29-0e59f2ffc438 None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:16:41.410902 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-c18ec6d5-21d7-4adc-9c29-0e59f2ffc438 None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:16:41.411264 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-c18ec6d5-21d7-4adc-9c29-0e59f2ffc438 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:16:41.411781 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-c18ec6d5-21d7-4adc-9c29-0e59f2ffc438 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:16:41.420713 n-d-853704-1 nova-conductor[120658]: ERROR nova.compute.rpcapi [None req-c18ec6d5-21d7-4adc-9c29-0e59f2ffc438 None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 18 23:16:41.421091 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-c18ec6d5-21d7-4adc-9c29-0e59f2ffc438 None None] Releasing lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 18 23:16:41.422030 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server [None req-c18ec6d5-21d7-4adc-9c29-0e59f2ffc438 None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 23:16:41.422030 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 23:16:41.422030 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 18 23:16:41.422030 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 18 23:16:41.422030 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 18 23:16:41.422030 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 23:16:41.422030 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 18 23:16:41.422030 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 23:16:41.422030 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 23:16:41.422030 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 18 23:16:41.422030 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 18 23:16:41.422030 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 18 23:16:41.422030 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 18 23:16:41.422030 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 18 23:16:41.422030 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 18 23:16:41.422030 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 18 23:16:41.422030 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 18 23:16:41.422030 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 18 23:16:41.422030 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 18 23:16:41.422030 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 18 23:16:41.422030 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 18 23:16:41.422030 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 18 23:16:41.422030 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 18 23:16:41.422030 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 23:16:41.422030 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 23:16:43.364113 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=6,nova_cell0:UPDATE=3 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:16:51.307612 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=13,nova_api:UPDATE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:16:51.350964 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=11,nova_api:DELETE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:16:51.417021 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:16:51.418715 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell1:SELECT=11,nova_cell1:SAVEPOINT=1,nova_cell1:INSERT=18,nova_cell1:RELEASE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:16:53.336559 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=14,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:16:55.347169 n-d-853704-1 nova-conductor[120657]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 18 23:17:23.361143 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:17:33.333519 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:17:47.537770 n-d-853704-1 nova-conductor[120657]: ERROR oslo.messaging._drivers.impl_rabbit [None req-82d17041-188b-47b5-8032-9ad7f0192275 tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] [ae71ad44-1ef2-45ec-82d0-4ccfc1149604] AMQP server on 10.222.0.49:5672 is unreachable: [Errno 104] Connection reset by peer. Trying again in 1 seconds.: ConnectionResetError: [Errno 104] Connection reset by peer Aug 18 23:17:48.545801 n-d-853704-1 nova-conductor[120657]: INFO oslo.messaging._drivers.impl_rabbit [None req-82d17041-188b-47b5-8032-9ad7f0192275 tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] [ae71ad44-1ef2-45ec-82d0-4ccfc1149604] Reconnected to AMQP server on 10.222.0.49:5672 via [amqp] client with port 45202. Aug 18 23:17:50.945774 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-82d17041-188b-47b5-8032-9ad7f0192275 tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Took 3.41 seconds to select destinations for 1 instance(s). {{(pid=120657) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 18 23:17:50.965510 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-82d17041-188b-47b5-8032-9ad7f0192275 tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:17:50.965950 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-82d17041-188b-47b5-8032-9ad7f0192275 tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:17:50.966347 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-82d17041-188b-47b5-8032-9ad7f0192275 tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:17:51.084762 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-82d17041-188b-47b5-8032-9ad7f0192275 tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:17:51.085604 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-82d17041-188b-47b5-8032-9ad7f0192275 tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:17:51.086118 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-82d17041-188b-47b5-8032-9ad7f0192275 tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:17:51.086663 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-82d17041-188b-47b5-8032-9ad7f0192275 tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:17:51.087067 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-82d17041-188b-47b5-8032-9ad7f0192275 tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:17:51.087453 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-82d17041-188b-47b5-8032-9ad7f0192275 tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:17:51.108049 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-82d17041-188b-47b5-8032-9ad7f0192275 tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Getting quotas for project 05baaf842a13444d8905683625e5853a. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 18 23:17:51.113560 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-82d17041-188b-47b5-8032-9ad7f0192275 tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Getting quotas for user 4d73c623dd454555ac8d974cf7a60398 and project 05baaf842a13444d8905683625e5853a. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 18 23:17:51.127431 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-82d17041-188b-47b5-8032-9ad7f0192275 tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] [instance: 52b4ed2e-f15f-4058-91ad-2b318e54c993] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120657) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 18 23:17:51.128279 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-82d17041-188b-47b5-8032-9ad7f0192275 tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:17:51.128724 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-82d17041-188b-47b5-8032-9ad7f0192275 tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:17:51.129112 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-82d17041-188b-47b5-8032-9ad7f0192275 tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:17:51.226946 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-82d17041-188b-47b5-8032-9ad7f0192275 tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] [instance: 52b4ed2e-f15f-4058-91ad-2b318e54c993] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='87b0fec9-a948-41f9-a3c0-f0a6a5a8f1e7',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120657) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 18 23:17:51.226946 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-82d17041-188b-47b5-8032-9ad7f0192275 tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:17:51.226946 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-82d17041-188b-47b5-8032-9ad7f0192275 tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:17:51.226946 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-82d17041-188b-47b5-8032-9ad7f0192275 tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:17:51.480388 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-82d17041-188b-47b5-8032-9ad7f0192275 tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:17:51.480843 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-82d17041-188b-47b5-8032-9ad7f0192275 tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:17:51.481235 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-82d17041-188b-47b5-8032-9ad7f0192275 tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:17:51.481691 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-82d17041-188b-47b5-8032-9ad7f0192275 tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Acquiring lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 18 23:17:51.482040 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-82d17041-188b-47b5-8032-9ad7f0192275 tempest-ServersTestJSON-619214383 tempest-ServersTestJSON-619214383-project] Acquired lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 18 23:17:51.482773 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-03fdea89-7d66-4bfa-a19f-734e169f5e84 None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:17:51.483185 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-03fdea89-7d66-4bfa-a19f-734e169f5e84 None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:17:51.483569 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-03fdea89-7d66-4bfa-a19f-734e169f5e84 None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:17:51.484268 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-03fdea89-7d66-4bfa-a19f-734e169f5e84 None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:17:51.484783 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-03fdea89-7d66-4bfa-a19f-734e169f5e84 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:17:51.485409 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-03fdea89-7d66-4bfa-a19f-734e169f5e84 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:17:51.493891 n-d-853704-1 nova-conductor[120657]: ERROR nova.compute.rpcapi [None req-03fdea89-7d66-4bfa-a19f-734e169f5e84 None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 18 23:17:51.494277 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-03fdea89-7d66-4bfa-a19f-734e169f5e84 None None] Releasing lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 18 23:17:51.495201 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server [None req-03fdea89-7d66-4bfa-a19f-734e169f5e84 None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 23:17:51.495201 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 23:17:51.495201 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 18 23:17:51.495201 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 18 23:17:51.495201 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 18 23:17:51.495201 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 23:17:51.495201 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 18 23:17:51.495201 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 23:17:51.495201 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 23:17:51.495201 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 18 23:17:51.495201 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 18 23:17:51.495201 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 18 23:17:51.495201 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 18 23:17:51.495201 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 18 23:17:51.495201 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 18 23:17:51.495201 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 18 23:17:51.495201 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 18 23:17:51.495201 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 18 23:17:51.495201 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 18 23:17:51.495201 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 18 23:17:51.495201 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 18 23:17:51.495201 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 18 23:17:51.495201 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 18 23:17:51.495201 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 23:17:51.495201 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 23:18:01.391386 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=11,nova_api:UPDATE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:18:01.392697 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=13,nova_api:DELETE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:18:01.491153 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:18:01.491991 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell1:SELECT=11,nova_cell1:INSERT=17 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:18:03.344023 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=6,nova_cell0:UPDATE=3 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:18:03.364847 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=14,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:18:43.352050 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:18:43.371288 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:19:03.356098 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=3,nova_cell0:UPDATE=2 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:19:13.373911 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=5,nova_cell0:UPDATE=3 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:19:33.387868 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=4,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:19:43.361590 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:20:13.381218 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:20:15.355866 n-d-853704-1 nova-conductor[120657]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 18 23:20:23.359842 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:20:53.381262 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:21:03.368899 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=7,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:21:23.392140 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=5,nova_cell0:UPDATE=3 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:21:41.318867 n-d-853704-1 nova-conductor[120658]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 18 23:21:43.372762 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:22:03.389344 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:22:23.386829 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=7,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:22:23.398407 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=3,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:22:53.030829 n-d-853704-1 nova-conductor[120657]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 18 23:23:03.396174 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:23:03.405363 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:23:20.707251 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-1b672e07-aab2-4cc7-b5cf-8bcfdb3da0ce tempest-AttachVolumeNegativeTest-144902305 tempest-AttachVolumeNegativeTest-144902305-project] Took 0.77 seconds to select destinations for 1 instance(s). {{(pid=120658) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 18 23:23:20.728822 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-1b672e07-aab2-4cc7-b5cf-8bcfdb3da0ce tempest-AttachVolumeNegativeTest-144902305 tempest-AttachVolumeNegativeTest-144902305-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:23:20.729263 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-1b672e07-aab2-4cc7-b5cf-8bcfdb3da0ce tempest-AttachVolumeNegativeTest-144902305 tempest-AttachVolumeNegativeTest-144902305-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:23:20.729654 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-1b672e07-aab2-4cc7-b5cf-8bcfdb3da0ce tempest-AttachVolumeNegativeTest-144902305 tempest-AttachVolumeNegativeTest-144902305-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:23:20.876320 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-1b672e07-aab2-4cc7-b5cf-8bcfdb3da0ce tempest-AttachVolumeNegativeTest-144902305 tempest-AttachVolumeNegativeTest-144902305-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:23:20.876752 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-1b672e07-aab2-4cc7-b5cf-8bcfdb3da0ce tempest-AttachVolumeNegativeTest-144902305 tempest-AttachVolumeNegativeTest-144902305-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:23:20.877138 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-1b672e07-aab2-4cc7-b5cf-8bcfdb3da0ce tempest-AttachVolumeNegativeTest-144902305 tempest-AttachVolumeNegativeTest-144902305-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:23:20.877785 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-1b672e07-aab2-4cc7-b5cf-8bcfdb3da0ce tempest-AttachVolumeNegativeTest-144902305 tempest-AttachVolumeNegativeTest-144902305-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:23:20.878192 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-1b672e07-aab2-4cc7-b5cf-8bcfdb3da0ce tempest-AttachVolumeNegativeTest-144902305 tempest-AttachVolumeNegativeTest-144902305-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:23:20.878607 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-1b672e07-aab2-4cc7-b5cf-8bcfdb3da0ce tempest-AttachVolumeNegativeTest-144902305 tempest-AttachVolumeNegativeTest-144902305-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:23:20.894369 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-1b672e07-aab2-4cc7-b5cf-8bcfdb3da0ce tempest-AttachVolumeNegativeTest-144902305 tempest-AttachVolumeNegativeTest-144902305-project] Getting quotas for project 666550d4121145498c79960dde4a67a0. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 18 23:23:20.899434 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-1b672e07-aab2-4cc7-b5cf-8bcfdb3da0ce tempest-AttachVolumeNegativeTest-144902305 tempest-AttachVolumeNegativeTest-144902305-project] Getting quotas for user adf3f0e43b41460797d29d4f20438f8f and project 666550d4121145498c79960dde4a67a0. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 18 23:23:20.910231 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-1b672e07-aab2-4cc7-b5cf-8bcfdb3da0ce tempest-AttachVolumeNegativeTest-144902305 tempest-AttachVolumeNegativeTest-144902305-project] [instance: e816dba0-c808-40fa-b873-af6c81f910b6] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120658) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 18 23:23:20.910989 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-1b672e07-aab2-4cc7-b5cf-8bcfdb3da0ce tempest-AttachVolumeNegativeTest-144902305 tempest-AttachVolumeNegativeTest-144902305-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:23:20.911411 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-1b672e07-aab2-4cc7-b5cf-8bcfdb3da0ce tempest-AttachVolumeNegativeTest-144902305 tempest-AttachVolumeNegativeTest-144902305-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:23:20.911836 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-1b672e07-aab2-4cc7-b5cf-8bcfdb3da0ce tempest-AttachVolumeNegativeTest-144902305 tempest-AttachVolumeNegativeTest-144902305-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:23:20.968876 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-1b672e07-aab2-4cc7-b5cf-8bcfdb3da0ce tempest-AttachVolumeNegativeTest-144902305 tempest-AttachVolumeNegativeTest-144902305-project] [instance: e816dba0-c808-40fa-b873-af6c81f910b6] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='87b0fec9-a948-41f9-a3c0-f0a6a5a8f1e7',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120658) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 18 23:23:20.970215 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-1b672e07-aab2-4cc7-b5cf-8bcfdb3da0ce tempest-AttachVolumeNegativeTest-144902305 tempest-AttachVolumeNegativeTest-144902305-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:23:20.970942 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-1b672e07-aab2-4cc7-b5cf-8bcfdb3da0ce tempest-AttachVolumeNegativeTest-144902305 tempest-AttachVolumeNegativeTest-144902305-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:23:20.971360 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-1b672e07-aab2-4cc7-b5cf-8bcfdb3da0ce tempest-AttachVolumeNegativeTest-144902305 tempest-AttachVolumeNegativeTest-144902305-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:23:21.197137 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-1b672e07-aab2-4cc7-b5cf-8bcfdb3da0ce tempest-AttachVolumeNegativeTest-144902305 tempest-AttachVolumeNegativeTest-144902305-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:23:21.197581 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-1b672e07-aab2-4cc7-b5cf-8bcfdb3da0ce tempest-AttachVolumeNegativeTest-144902305 tempest-AttachVolumeNegativeTest-144902305-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:23:21.197976 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-1b672e07-aab2-4cc7-b5cf-8bcfdb3da0ce tempest-AttachVolumeNegativeTest-144902305 tempest-AttachVolumeNegativeTest-144902305-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:23:21.198433 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-1b672e07-aab2-4cc7-b5cf-8bcfdb3da0ce tempest-AttachVolumeNegativeTest-144902305 tempest-AttachVolumeNegativeTest-144902305-project] Acquiring lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 18 23:23:21.198800 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-1b672e07-aab2-4cc7-b5cf-8bcfdb3da0ce tempest-AttachVolumeNegativeTest-144902305 tempest-AttachVolumeNegativeTest-144902305-project] Acquired lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 18 23:23:21.199533 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-bdbb76ef-ad85-4d4f-8048-0fbdd6919c71 None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:23:21.199983 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-bdbb76ef-ad85-4d4f-8048-0fbdd6919c71 None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:23:21.203710 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-bdbb76ef-ad85-4d4f-8048-0fbdd6919c71 None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:23:21.203710 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-bdbb76ef-ad85-4d4f-8048-0fbdd6919c71 None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:23:21.203710 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-bdbb76ef-ad85-4d4f-8048-0fbdd6919c71 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:23:21.203710 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-bdbb76ef-ad85-4d4f-8048-0fbdd6919c71 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:23:21.214585 n-d-853704-1 nova-conductor[120658]: ERROR nova.compute.rpcapi [None req-bdbb76ef-ad85-4d4f-8048-0fbdd6919c71 None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 18 23:23:21.214997 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-bdbb76ef-ad85-4d4f-8048-0fbdd6919c71 None None] Releasing lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 18 23:23:21.215922 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server [None req-bdbb76ef-ad85-4d4f-8048-0fbdd6919c71 None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 23:23:21.215922 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 23:23:21.215922 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 18 23:23:21.215922 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 18 23:23:21.215922 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 18 23:23:21.215922 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 23:23:21.215922 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 18 23:23:21.215922 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 23:23:21.215922 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 23:23:21.215922 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 18 23:23:21.215922 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 18 23:23:21.215922 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 18 23:23:21.215922 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 18 23:23:21.215922 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 18 23:23:21.215922 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 18 23:23:21.215922 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 18 23:23:21.215922 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 18 23:23:21.215922 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 18 23:23:21.215922 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 18 23:23:21.215922 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 18 23:23:21.215922 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 18 23:23:21.215922 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 18 23:23:21.215922 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 18 23:23:21.215922 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 23:23:21.215922 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 23:23:31.046283 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=13 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:23:31.112393 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=11,nova_api:UPDATE=1,nova_api:DELETE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:23:31.212747 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:23:31.215675 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell1:SELECT=11,nova_cell1:SAVEPOINT=1,nova_cell1:INSERT=18,nova_cell1:RELEASE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:23:35.368967 n-d-853704-1 nova-conductor[120657]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 18 23:23:43.397662 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=15,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:23:43.410152 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:24:23.397392 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:24:23.408697 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:24:32.679333 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-3a20dd22-26d4-4d2d-b0a8-52b68190b99b tempest-VolumesActionsTest-719160834 tempest-VolumesActionsTest-719160834-project] Took 0.73 seconds to select destinations for 1 instance(s). {{(pid=120657) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 18 23:24:32.698906 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-3a20dd22-26d4-4d2d-b0a8-52b68190b99b tempest-VolumesActionsTest-719160834 tempest-VolumesActionsTest-719160834-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:24:32.699350 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-3a20dd22-26d4-4d2d-b0a8-52b68190b99b tempest-VolumesActionsTest-719160834 tempest-VolumesActionsTest-719160834-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:24:32.699768 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-3a20dd22-26d4-4d2d-b0a8-52b68190b99b tempest-VolumesActionsTest-719160834 tempest-VolumesActionsTest-719160834-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:24:32.973266 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-3a20dd22-26d4-4d2d-b0a8-52b68190b99b tempest-VolumesActionsTest-719160834 tempest-VolumesActionsTest-719160834-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:24:32.973711 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-3a20dd22-26d4-4d2d-b0a8-52b68190b99b tempest-VolumesActionsTest-719160834 tempest-VolumesActionsTest-719160834-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:24:32.974110 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-3a20dd22-26d4-4d2d-b0a8-52b68190b99b tempest-VolumesActionsTest-719160834 tempest-VolumesActionsTest-719160834-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:24:32.974778 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-3a20dd22-26d4-4d2d-b0a8-52b68190b99b tempest-VolumesActionsTest-719160834 tempest-VolumesActionsTest-719160834-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:24:32.975193 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-3a20dd22-26d4-4d2d-b0a8-52b68190b99b tempest-VolumesActionsTest-719160834 tempest-VolumesActionsTest-719160834-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:24:32.975579 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-3a20dd22-26d4-4d2d-b0a8-52b68190b99b tempest-VolumesActionsTest-719160834 tempest-VolumesActionsTest-719160834-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:24:32.991180 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-3a20dd22-26d4-4d2d-b0a8-52b68190b99b tempest-VolumesActionsTest-719160834 tempest-VolumesActionsTest-719160834-project] Getting quotas for project 5eac627d37184b1cbf6fadd3467d30a6. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 18 23:24:32.995623 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-3a20dd22-26d4-4d2d-b0a8-52b68190b99b tempest-VolumesActionsTest-719160834 tempest-VolumesActionsTest-719160834-project] Getting quotas for user 5aee7c506ef54999bce9af251d3f6a93 and project 5eac627d37184b1cbf6fadd3467d30a6. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 18 23:24:33.006722 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-3a20dd22-26d4-4d2d-b0a8-52b68190b99b tempest-VolumesActionsTest-719160834 tempest-VolumesActionsTest-719160834-project] [instance: 52026172-07a4-49a1-85da-7dd9c5c8044c] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120657) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 18 23:24:33.007551 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-3a20dd22-26d4-4d2d-b0a8-52b68190b99b tempest-VolumesActionsTest-719160834 tempest-VolumesActionsTest-719160834-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:24:33.007993 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-3a20dd22-26d4-4d2d-b0a8-52b68190b99b tempest-VolumesActionsTest-719160834 tempest-VolumesActionsTest-719160834-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:24:33.008392 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-3a20dd22-26d4-4d2d-b0a8-52b68190b99b tempest-VolumesActionsTest-719160834 tempest-VolumesActionsTest-719160834-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:24:33.081247 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-3a20dd22-26d4-4d2d-b0a8-52b68190b99b tempest-VolumesActionsTest-719160834 tempest-VolumesActionsTest-719160834-project] [instance: 52026172-07a4-49a1-85da-7dd9c5c8044c] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='87b0fec9-a948-41f9-a3c0-f0a6a5a8f1e7',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120657) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 18 23:24:33.082543 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-3a20dd22-26d4-4d2d-b0a8-52b68190b99b tempest-VolumesActionsTest-719160834 tempest-VolumesActionsTest-719160834-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:24:33.082985 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-3a20dd22-26d4-4d2d-b0a8-52b68190b99b tempest-VolumesActionsTest-719160834 tempest-VolumesActionsTest-719160834-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:24:33.083633 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-3a20dd22-26d4-4d2d-b0a8-52b68190b99b tempest-VolumesActionsTest-719160834 tempest-VolumesActionsTest-719160834-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:24:33.409896 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-3a20dd22-26d4-4d2d-b0a8-52b68190b99b tempest-VolumesActionsTest-719160834 tempest-VolumesActionsTest-719160834-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:24:33.410343 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-3a20dd22-26d4-4d2d-b0a8-52b68190b99b tempest-VolumesActionsTest-719160834 tempest-VolumesActionsTest-719160834-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:24:33.410736 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-3a20dd22-26d4-4d2d-b0a8-52b68190b99b tempest-VolumesActionsTest-719160834 tempest-VolumesActionsTest-719160834-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:24:33.411191 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-3a20dd22-26d4-4d2d-b0a8-52b68190b99b tempest-VolumesActionsTest-719160834 tempest-VolumesActionsTest-719160834-project] Acquiring lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 18 23:24:33.411544 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-3a20dd22-26d4-4d2d-b0a8-52b68190b99b tempest-VolumesActionsTest-719160834 tempest-VolumesActionsTest-719160834-project] Acquired lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 18 23:24:33.412306 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-54971cfd-3ef4-450e-acf6-910d8e8df2a4 None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:24:33.412719 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-54971cfd-3ef4-450e-acf6-910d8e8df2a4 None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:24:33.413098 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-54971cfd-3ef4-450e-acf6-910d8e8df2a4 None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:24:33.413743 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-54971cfd-3ef4-450e-acf6-910d8e8df2a4 None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:24:33.414148 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-54971cfd-3ef4-450e-acf6-910d8e8df2a4 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:24:33.414524 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-54971cfd-3ef4-450e-acf6-910d8e8df2a4 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:24:33.423701 n-d-853704-1 nova-conductor[120657]: ERROR nova.compute.rpcapi [None req-54971cfd-3ef4-450e-acf6-910d8e8df2a4 None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 18 23:24:33.424080 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-54971cfd-3ef4-450e-acf6-910d8e8df2a4 None None] Releasing lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 18 23:24:33.424993 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server [None req-54971cfd-3ef4-450e-acf6-910d8e8df2a4 None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 23:24:33.424993 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 23:24:33.424993 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 18 23:24:33.424993 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 18 23:24:33.424993 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 18 23:24:33.424993 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 23:24:33.424993 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 18 23:24:33.424993 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 23:24:33.424993 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 23:24:33.424993 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 18 23:24:33.424993 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 18 23:24:33.424993 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 18 23:24:33.424993 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 18 23:24:33.424993 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 18 23:24:33.424993 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 18 23:24:33.424993 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 18 23:24:33.424993 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 18 23:24:33.424993 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 18 23:24:33.424993 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 18 23:24:33.424993 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 18 23:24:33.424993 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 18 23:24:33.424993 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 18 23:24:33.424993 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 18 23:24:33.424993 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 23:24:33.424993 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 23:24:43.301981 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=11 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:24:43.302763 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=13,nova_api:UPDATE=1,nova_api:DELETE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:24:43.421833 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:24:43.422638 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell1:SELECT=11,nova_cell1:SAVEPOINT=1,nova_cell1:INSERT=18,nova_cell1:RELEASE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:25:03.394839 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:25:03.404946 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=14,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:25:23.408436 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=3,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:25:40.011194 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-7b91816d-059b-4556-b0c0-6f2f6ae8a003 tempest-TestVolumeBootPattern-1034437477 tempest-TestVolumeBootPattern-1034437477-project] Took 0.42 seconds to select destinations for 1 instance(s). {{(pid=120658) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 18 23:25:40.032008 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-7b91816d-059b-4556-b0c0-6f2f6ae8a003 tempest-TestVolumeBootPattern-1034437477 tempest-TestVolumeBootPattern-1034437477-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:25:40.037483 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-7b91816d-059b-4556-b0c0-6f2f6ae8a003 tempest-TestVolumeBootPattern-1034437477 tempest-TestVolumeBootPattern-1034437477-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:25:40.037483 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-7b91816d-059b-4556-b0c0-6f2f6ae8a003 tempest-TestVolumeBootPattern-1034437477 tempest-TestVolumeBootPattern-1034437477-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:25:40.151465 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:25:40.153825 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-7b91816d-059b-4556-b0c0-6f2f6ae8a003 tempest-TestVolumeBootPattern-1034437477 tempest-TestVolumeBootPattern-1034437477-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:25:40.154267 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-7b91816d-059b-4556-b0c0-6f2f6ae8a003 tempest-TestVolumeBootPattern-1034437477 tempest-TestVolumeBootPattern-1034437477-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:25:40.154866 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-7b91816d-059b-4556-b0c0-6f2f6ae8a003 tempest-TestVolumeBootPattern-1034437477 tempest-TestVolumeBootPattern-1034437477-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:25:40.155538 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-7b91816d-059b-4556-b0c0-6f2f6ae8a003 tempest-TestVolumeBootPattern-1034437477 tempest-TestVolumeBootPattern-1034437477-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:25:40.155984 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-7b91816d-059b-4556-b0c0-6f2f6ae8a003 tempest-TestVolumeBootPattern-1034437477 tempest-TestVolumeBootPattern-1034437477-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:25:40.156368 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-7b91816d-059b-4556-b0c0-6f2f6ae8a003 tempest-TestVolumeBootPattern-1034437477 tempest-TestVolumeBootPattern-1034437477-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:25:40.177076 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-7b91816d-059b-4556-b0c0-6f2f6ae8a003 tempest-TestVolumeBootPattern-1034437477 tempest-TestVolumeBootPattern-1034437477-project] Getting quotas for project dfab7231c21e4b1f82cb2fbf1c4e0986. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 18 23:25:40.183497 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-7b91816d-059b-4556-b0c0-6f2f6ae8a003 tempest-TestVolumeBootPattern-1034437477 tempest-TestVolumeBootPattern-1034437477-project] Getting quotas for user 480cd2375ac24f8a89b9730fcaa6965e and project dfab7231c21e4b1f82cb2fbf1c4e0986. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 18 23:25:40.194877 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-7b91816d-059b-4556-b0c0-6f2f6ae8a003 tempest-TestVolumeBootPattern-1034437477 tempest-TestVolumeBootPattern-1034437477-project] [instance: bfd71fc6-4f79-4892-9375-bd74bc29cd60] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120658) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 18 23:25:40.195615 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-7b91816d-059b-4556-b0c0-6f2f6ae8a003 tempest-TestVolumeBootPattern-1034437477 tempest-TestVolumeBootPattern-1034437477-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:25:40.196115 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-7b91816d-059b-4556-b0c0-6f2f6ae8a003 tempest-TestVolumeBootPattern-1034437477 tempest-TestVolumeBootPattern-1034437477-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:25:40.196511 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-7b91816d-059b-4556-b0c0-6f2f6ae8a003 tempest-TestVolumeBootPattern-1034437477 tempest-TestVolumeBootPattern-1034437477-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:25:40.279835 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-7b91816d-059b-4556-b0c0-6f2f6ae8a003 tempest-TestVolumeBootPattern-1034437477 tempest-TestVolumeBootPattern-1034437477-project] [instance: bfd71fc6-4f79-4892-9375-bd74bc29cd60] block_device_mapping [BlockDeviceMapping(attachment_id=88d03005-3825-4787-85af-78e98945f090,boot_index=0,connection_info=None,created_at=,delete_on_termination=False,deleted=,deleted_at=,destination_type='volume',device_name=None,device_type=None,disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id=None,instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='volume',tag=None,updated_at=,uuid=,volume_id='bb6a9532-37b6-4d2d-9898-801af5597fd3',volume_size=1,volume_type=None)] {{(pid=120658) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 18 23:25:40.284817 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-7b91816d-059b-4556-b0c0-6f2f6ae8a003 tempest-TestVolumeBootPattern-1034437477 tempest-TestVolumeBootPattern-1034437477-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:25:40.284817 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-7b91816d-059b-4556-b0c0-6f2f6ae8a003 tempest-TestVolumeBootPattern-1034437477 tempest-TestVolumeBootPattern-1034437477-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:25:40.284817 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-7b91816d-059b-4556-b0c0-6f2f6ae8a003 tempest-TestVolumeBootPattern-1034437477 tempest-TestVolumeBootPattern-1034437477-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:25:40.615291 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-7b91816d-059b-4556-b0c0-6f2f6ae8a003 tempest-TestVolumeBootPattern-1034437477 tempest-TestVolumeBootPattern-1034437477-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:25:40.615782 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-7b91816d-059b-4556-b0c0-6f2f6ae8a003 tempest-TestVolumeBootPattern-1034437477 tempest-TestVolumeBootPattern-1034437477-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:25:40.616180 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-7b91816d-059b-4556-b0c0-6f2f6ae8a003 tempest-TestVolumeBootPattern-1034437477 tempest-TestVolumeBootPattern-1034437477-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:25:40.616632 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-7b91816d-059b-4556-b0c0-6f2f6ae8a003 tempest-TestVolumeBootPattern-1034437477 tempest-TestVolumeBootPattern-1034437477-project] Acquiring lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 18 23:25:40.619488 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-7b91816d-059b-4556-b0c0-6f2f6ae8a003 tempest-TestVolumeBootPattern-1034437477 tempest-TestVolumeBootPattern-1034437477-project] Acquired lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 18 23:25:40.619488 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-368c156b-4d81-4fa0-b416-853934886765 None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:25:40.619488 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-368c156b-4d81-4fa0-b416-853934886765 None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:25:40.619488 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-368c156b-4d81-4fa0-b416-853934886765 None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:25:40.619488 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-368c156b-4d81-4fa0-b416-853934886765 None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:25:40.620013 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-368c156b-4d81-4fa0-b416-853934886765 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:25:40.620183 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-368c156b-4d81-4fa0-b416-853934886765 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:25:40.629426 n-d-853704-1 nova-conductor[120658]: ERROR nova.compute.rpcapi [None req-368c156b-4d81-4fa0-b416-853934886765 None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 18 23:25:40.629921 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-368c156b-4d81-4fa0-b416-853934886765 None None] Releasing lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 18 23:25:40.630844 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server [None req-368c156b-4d81-4fa0-b416-853934886765 None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 23:25:40.630844 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 23:25:40.630844 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 18 23:25:40.630844 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 18 23:25:40.630844 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 18 23:25:40.630844 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 23:25:40.630844 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 18 23:25:40.630844 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 23:25:40.630844 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 23:25:40.630844 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 18 23:25:40.630844 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 18 23:25:40.630844 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 18 23:25:40.630844 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 18 23:25:40.630844 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 18 23:25:40.630844 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 18 23:25:40.630844 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 18 23:25:40.630844 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 18 23:25:40.630844 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 18 23:25:40.630844 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 18 23:25:40.630844 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 18 23:25:40.630844 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 18 23:25:40.630844 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 18 23:25:40.630844 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 18 23:25:40.630844 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 23:25:40.630844 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 23:25:50.472524 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=10,nova_api:UPDATE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:25:50.473752 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=14,nova_api:DELETE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:25:50.627366 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:25:50.628436 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell1:SELECT=11,nova_cell1:INSERT=11 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:26:03.404592 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:26:13.393521 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=12,nova_cell0:UPDATE=3 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:26:23.404956 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=3,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:26:53.402525 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:26:55.378483 n-d-853704-1 nova-conductor[120657]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 18 23:27:03.412550 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:27:23.405319 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=5,nova_cell0:UPDATE=3 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:27:43.412750 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:28:03.413882 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:28:21.330737 n-d-853704-1 nova-conductor[120658]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 18 23:28:23.417815 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:28:43.421906 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=3,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:28:43.423562 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=7,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:29:23.414775 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:29:23.415569 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:29:27.399266 n-d-853704-1 nova-conductor[120657]: ERROR oslo.messaging._drivers.impl_rabbit [None req-649761db-d6d4-4a92-819a-a56df98a353f tempest-AttachVolumeNegativeTest-144902305 tempest-AttachVolumeNegativeTest-144902305-project] [ae71ad44-1ef2-45ec-82d0-4ccfc1149604] AMQP server on 10.222.0.49:5672 is unreachable: [Errno 104] Connection reset by peer. Trying again in 1 seconds.: ConnectionResetError: [Errno 104] Connection reset by peer Aug 18 23:29:28.407316 n-d-853704-1 nova-conductor[120657]: INFO oslo.messaging._drivers.impl_rabbit [None req-649761db-d6d4-4a92-819a-a56df98a353f tempest-AttachVolumeNegativeTest-144902305 tempest-AttachVolumeNegativeTest-144902305-project] [ae71ad44-1ef2-45ec-82d0-4ccfc1149604] Reconnected to AMQP server on 10.222.0.49:5672 via [amqp] client with port 57342. Aug 18 23:29:30.811694 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-649761db-d6d4-4a92-819a-a56df98a353f tempest-AttachVolumeNegativeTest-144902305 tempest-AttachVolumeNegativeTest-144902305-project] Took 3.42 seconds to select destinations for 1 instance(s). {{(pid=120657) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 18 23:29:30.832303 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-649761db-d6d4-4a92-819a-a56df98a353f tempest-AttachVolumeNegativeTest-144902305 tempest-AttachVolumeNegativeTest-144902305-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:29:30.832747 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-649761db-d6d4-4a92-819a-a56df98a353f tempest-AttachVolumeNegativeTest-144902305 tempest-AttachVolumeNegativeTest-144902305-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:29:30.833169 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-649761db-d6d4-4a92-819a-a56df98a353f tempest-AttachVolumeNegativeTest-144902305 tempest-AttachVolumeNegativeTest-144902305-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:29:31.010662 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-649761db-d6d4-4a92-819a-a56df98a353f tempest-AttachVolumeNegativeTest-144902305 tempest-AttachVolumeNegativeTest-144902305-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:29:31.011142 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-649761db-d6d4-4a92-819a-a56df98a353f tempest-AttachVolumeNegativeTest-144902305 tempest-AttachVolumeNegativeTest-144902305-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:29:31.011537 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-649761db-d6d4-4a92-819a-a56df98a353f tempest-AttachVolumeNegativeTest-144902305 tempest-AttachVolumeNegativeTest-144902305-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:29:31.012225 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-649761db-d6d4-4a92-819a-a56df98a353f tempest-AttachVolumeNegativeTest-144902305 tempest-AttachVolumeNegativeTest-144902305-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:29:31.012634 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-649761db-d6d4-4a92-819a-a56df98a353f tempest-AttachVolumeNegativeTest-144902305 tempest-AttachVolumeNegativeTest-144902305-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:29:31.013016 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-649761db-d6d4-4a92-819a-a56df98a353f tempest-AttachVolumeNegativeTest-144902305 tempest-AttachVolumeNegativeTest-144902305-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:29:31.030144 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-649761db-d6d4-4a92-819a-a56df98a353f tempest-AttachVolumeNegativeTest-144902305 tempest-AttachVolumeNegativeTest-144902305-project] Getting quotas for project 666550d4121145498c79960dde4a67a0. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 18 23:29:31.038337 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-649761db-d6d4-4a92-819a-a56df98a353f tempest-AttachVolumeNegativeTest-144902305 tempest-AttachVolumeNegativeTest-144902305-project] Getting quotas for user adf3f0e43b41460797d29d4f20438f8f and project 666550d4121145498c79960dde4a67a0. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 18 23:29:31.053335 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-649761db-d6d4-4a92-819a-a56df98a353f tempest-AttachVolumeNegativeTest-144902305 tempest-AttachVolumeNegativeTest-144902305-project] [instance: 67116556-92a7-4c6d-a724-4a51f437f287] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120657) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 18 23:29:31.054170 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-649761db-d6d4-4a92-819a-a56df98a353f tempest-AttachVolumeNegativeTest-144902305 tempest-AttachVolumeNegativeTest-144902305-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:29:31.054610 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-649761db-d6d4-4a92-819a-a56df98a353f tempest-AttachVolumeNegativeTest-144902305 tempest-AttachVolumeNegativeTest-144902305-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:29:31.055003 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-649761db-d6d4-4a92-819a-a56df98a353f tempest-AttachVolumeNegativeTest-144902305 tempest-AttachVolumeNegativeTest-144902305-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:29:31.148053 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-649761db-d6d4-4a92-819a-a56df98a353f tempest-AttachVolumeNegativeTest-144902305 tempest-AttachVolumeNegativeTest-144902305-project] [instance: 67116556-92a7-4c6d-a724-4a51f437f287] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='87b0fec9-a948-41f9-a3c0-f0a6a5a8f1e7',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120657) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 18 23:29:31.149351 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-649761db-d6d4-4a92-819a-a56df98a353f tempest-AttachVolumeNegativeTest-144902305 tempest-AttachVolumeNegativeTest-144902305-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:29:31.149820 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-649761db-d6d4-4a92-819a-a56df98a353f tempest-AttachVolumeNegativeTest-144902305 tempest-AttachVolumeNegativeTest-144902305-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:29:31.150210 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-649761db-d6d4-4a92-819a-a56df98a353f tempest-AttachVolumeNegativeTest-144902305 tempest-AttachVolumeNegativeTest-144902305-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:29:31.480300 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-649761db-d6d4-4a92-819a-a56df98a353f tempest-AttachVolumeNegativeTest-144902305 tempest-AttachVolumeNegativeTest-144902305-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:29:31.480740 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-649761db-d6d4-4a92-819a-a56df98a353f tempest-AttachVolumeNegativeTest-144902305 tempest-AttachVolumeNegativeTest-144902305-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:29:31.481156 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-649761db-d6d4-4a92-819a-a56df98a353f tempest-AttachVolumeNegativeTest-144902305 tempest-AttachVolumeNegativeTest-144902305-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:29:31.481605 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-649761db-d6d4-4a92-819a-a56df98a353f tempest-AttachVolumeNegativeTest-144902305 tempest-AttachVolumeNegativeTest-144902305-project] Acquiring lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 18 23:29:31.481954 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-649761db-d6d4-4a92-819a-a56df98a353f tempest-AttachVolumeNegativeTest-144902305 tempest-AttachVolumeNegativeTest-144902305-project] Acquired lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 18 23:29:31.482680 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-4d731103-66ec-48c4-beee-ceb36dbd97eb None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:29:31.483086 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-4d731103-66ec-48c4-beee-ceb36dbd97eb None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:29:31.483464 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-4d731103-66ec-48c4-beee-ceb36dbd97eb None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:29:31.484141 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-4d731103-66ec-48c4-beee-ceb36dbd97eb None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:29:31.484631 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-4d731103-66ec-48c4-beee-ceb36dbd97eb None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:29:31.485072 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-4d731103-66ec-48c4-beee-ceb36dbd97eb None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:29:31.495231 n-d-853704-1 nova-conductor[120657]: ERROR nova.compute.rpcapi [None req-4d731103-66ec-48c4-beee-ceb36dbd97eb None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 18 23:29:31.496877 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-4d731103-66ec-48c4-beee-ceb36dbd97eb None None] Releasing lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 18 23:29:31.496877 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server [None req-4d731103-66ec-48c4-beee-ceb36dbd97eb None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 23:29:31.496877 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 23:29:31.496877 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 18 23:29:31.496877 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 18 23:29:31.496877 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 18 23:29:31.496877 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 23:29:31.496877 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 18 23:29:31.496877 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 23:29:31.496877 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 23:29:31.496877 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 18 23:29:31.496877 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 18 23:29:31.496877 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 18 23:29:31.496877 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 18 23:29:31.496877 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 18 23:29:31.496877 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 18 23:29:31.496877 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 18 23:29:31.496877 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 18 23:29:31.496877 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 18 23:29:31.496877 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 18 23:29:31.496877 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 18 23:29:31.496877 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 18 23:29:31.496877 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 18 23:29:31.496877 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 18 23:29:31.496877 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 23:29:31.496877 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 23:29:41.379773 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=12 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:29:41.381311 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=12,nova_api:UPDATE=1,nova_api:DELETE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:29:41.494636 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:29:41.495368 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell1:SELECT=11,nova_cell1:INSERT=17 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:30:03.403308 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:30:03.403947 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=14,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:30:15.387032 n-d-853704-1 nova-conductor[120657]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 18 23:30:23.411508 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=3,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:30:34.868620 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-3775a0e2-0f1d-40ae-b5aa-2adf12fa2bc9 tempest-VolumesActionsTest-719160834 tempest-VolumesActionsTest-719160834-project] Took 0.52 seconds to select destinations for 1 instance(s). {{(pid=120658) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 18 23:30:34.888687 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-3775a0e2-0f1d-40ae-b5aa-2adf12fa2bc9 tempest-VolumesActionsTest-719160834 tempest-VolumesActionsTest-719160834-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:30:34.889126 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-3775a0e2-0f1d-40ae-b5aa-2adf12fa2bc9 tempest-VolumesActionsTest-719160834 tempest-VolumesActionsTest-719160834-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:30:34.889517 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-3775a0e2-0f1d-40ae-b5aa-2adf12fa2bc9 tempest-VolumesActionsTest-719160834 tempest-VolumesActionsTest-719160834-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:30:34.985398 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:30:34.987650 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-3775a0e2-0f1d-40ae-b5aa-2adf12fa2bc9 tempest-VolumesActionsTest-719160834 tempest-VolumesActionsTest-719160834-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:30:34.988119 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-3775a0e2-0f1d-40ae-b5aa-2adf12fa2bc9 tempest-VolumesActionsTest-719160834 tempest-VolumesActionsTest-719160834-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:30:34.988512 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-3775a0e2-0f1d-40ae-b5aa-2adf12fa2bc9 tempest-VolumesActionsTest-719160834 tempest-VolumesActionsTest-719160834-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:30:34.989175 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-3775a0e2-0f1d-40ae-b5aa-2adf12fa2bc9 tempest-VolumesActionsTest-719160834 tempest-VolumesActionsTest-719160834-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:30:34.989632 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-3775a0e2-0f1d-40ae-b5aa-2adf12fa2bc9 tempest-VolumesActionsTest-719160834 tempest-VolumesActionsTest-719160834-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:30:34.989969 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-3775a0e2-0f1d-40ae-b5aa-2adf12fa2bc9 tempest-VolumesActionsTest-719160834 tempest-VolumesActionsTest-719160834-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:30:35.006112 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-3775a0e2-0f1d-40ae-b5aa-2adf12fa2bc9 tempest-VolumesActionsTest-719160834 tempest-VolumesActionsTest-719160834-project] Getting quotas for project 5eac627d37184b1cbf6fadd3467d30a6. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 18 23:30:35.010669 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-3775a0e2-0f1d-40ae-b5aa-2adf12fa2bc9 tempest-VolumesActionsTest-719160834 tempest-VolumesActionsTest-719160834-project] Getting quotas for user 5aee7c506ef54999bce9af251d3f6a93 and project 5eac627d37184b1cbf6fadd3467d30a6. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 18 23:30:35.022660 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-3775a0e2-0f1d-40ae-b5aa-2adf12fa2bc9 tempest-VolumesActionsTest-719160834 tempest-VolumesActionsTest-719160834-project] [instance: 125c3f93-3300-4ed4-9932-136c77583f13] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120658) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 18 23:30:35.023438 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-3775a0e2-0f1d-40ae-b5aa-2adf12fa2bc9 tempest-VolumesActionsTest-719160834 tempest-VolumesActionsTest-719160834-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:30:35.023971 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-3775a0e2-0f1d-40ae-b5aa-2adf12fa2bc9 tempest-VolumesActionsTest-719160834 tempest-VolumesActionsTest-719160834-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:30:35.024359 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-3775a0e2-0f1d-40ae-b5aa-2adf12fa2bc9 tempest-VolumesActionsTest-719160834 tempest-VolumesActionsTest-719160834-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:30:35.061781 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-3775a0e2-0f1d-40ae-b5aa-2adf12fa2bc9 tempest-VolumesActionsTest-719160834 tempest-VolumesActionsTest-719160834-project] [instance: 125c3f93-3300-4ed4-9932-136c77583f13] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='87b0fec9-a948-41f9-a3c0-f0a6a5a8f1e7',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120658) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 18 23:30:35.063177 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-3775a0e2-0f1d-40ae-b5aa-2adf12fa2bc9 tempest-VolumesActionsTest-719160834 tempest-VolumesActionsTest-719160834-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:30:35.063614 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-3775a0e2-0f1d-40ae-b5aa-2adf12fa2bc9 tempest-VolumesActionsTest-719160834 tempest-VolumesActionsTest-719160834-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:30:35.064036 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-3775a0e2-0f1d-40ae-b5aa-2adf12fa2bc9 tempest-VolumesActionsTest-719160834 tempest-VolumesActionsTest-719160834-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:30:35.251659 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-3775a0e2-0f1d-40ae-b5aa-2adf12fa2bc9 tempest-VolumesActionsTest-719160834 tempest-VolumesActionsTest-719160834-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:30:35.252121 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-3775a0e2-0f1d-40ae-b5aa-2adf12fa2bc9 tempest-VolumesActionsTest-719160834 tempest-VolumesActionsTest-719160834-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:30:35.252512 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-3775a0e2-0f1d-40ae-b5aa-2adf12fa2bc9 tempest-VolumesActionsTest-719160834 tempest-VolumesActionsTest-719160834-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:30:35.252986 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-3775a0e2-0f1d-40ae-b5aa-2adf12fa2bc9 tempest-VolumesActionsTest-719160834 tempest-VolumesActionsTest-719160834-project] Acquiring lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 18 23:30:35.253329 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-3775a0e2-0f1d-40ae-b5aa-2adf12fa2bc9 tempest-VolumesActionsTest-719160834 tempest-VolumesActionsTest-719160834-project] Acquired lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 18 23:30:35.254069 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-3a211012-57f6-4e6d-89c9-a7caaacfd8d1 None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:30:35.254480 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-3a211012-57f6-4e6d-89c9-a7caaacfd8d1 None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:30:35.254861 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-3a211012-57f6-4e6d-89c9-a7caaacfd8d1 None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:30:35.255546 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-3a211012-57f6-4e6d-89c9-a7caaacfd8d1 None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:30:35.256007 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-3a211012-57f6-4e6d-89c9-a7caaacfd8d1 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:30:35.256381 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-3a211012-57f6-4e6d-89c9-a7caaacfd8d1 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:30:35.267868 n-d-853704-1 nova-conductor[120658]: ERROR nova.compute.rpcapi [None req-3a211012-57f6-4e6d-89c9-a7caaacfd8d1 None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 18 23:30:35.268252 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-3a211012-57f6-4e6d-89c9-a7caaacfd8d1 None None] Releasing lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 18 23:30:35.269158 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server [None req-3a211012-57f6-4e6d-89c9-a7caaacfd8d1 None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 23:30:35.269158 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 23:30:35.269158 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 18 23:30:35.269158 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 18 23:30:35.269158 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 18 23:30:35.269158 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 23:30:35.269158 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 18 23:30:35.269158 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 23:30:35.269158 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 23:30:35.269158 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 18 23:30:35.269158 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 18 23:30:35.269158 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 18 23:30:35.269158 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 18 23:30:35.269158 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 18 23:30:35.269158 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 18 23:30:35.269158 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 18 23:30:35.269158 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 18 23:30:35.269158 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 18 23:30:35.269158 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 18 23:30:35.269158 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 18 23:30:35.269158 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 18 23:30:35.269158 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 18 23:30:35.269158 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 18 23:30:35.269158 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 23:30:35.269158 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 23:30:43.420739 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=4,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:30:45.129063 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=13 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:30:45.195726 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=11,nova_api:UPDATE=1,nova_api:DELETE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:30:45.266625 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:30:45.267301 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell1:SELECT=11,nova_cell1:INSERT=17 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:31:05.116464 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-46485b9d-d213-446e-8d95-c7704dce01c5 tempest-TestNetworkAdvancedServerOps-1005354344 tempest-TestNetworkAdvancedServerOps-1005354344-project] Took 0.76 seconds to select destinations for 1 instance(s). {{(pid=120657) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 18 23:31:05.160524 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-46485b9d-d213-446e-8d95-c7704dce01c5 tempest-TestNetworkAdvancedServerOps-1005354344 tempest-TestNetworkAdvancedServerOps-1005354344-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:31:05.160993 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-46485b9d-d213-446e-8d95-c7704dce01c5 tempest-TestNetworkAdvancedServerOps-1005354344 tempest-TestNetworkAdvancedServerOps-1005354344-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:31:05.161394 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-46485b9d-d213-446e-8d95-c7704dce01c5 tempest-TestNetworkAdvancedServerOps-1005354344 tempest-TestNetworkAdvancedServerOps-1005354344-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:31:05.342087 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-46485b9d-d213-446e-8d95-c7704dce01c5 tempest-TestNetworkAdvancedServerOps-1005354344 tempest-TestNetworkAdvancedServerOps-1005354344-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:31:05.342535 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-46485b9d-d213-446e-8d95-c7704dce01c5 tempest-TestNetworkAdvancedServerOps-1005354344 tempest-TestNetworkAdvancedServerOps-1005354344-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:31:05.342932 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-46485b9d-d213-446e-8d95-c7704dce01c5 tempest-TestNetworkAdvancedServerOps-1005354344 tempest-TestNetworkAdvancedServerOps-1005354344-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:31:05.343605 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-46485b9d-d213-446e-8d95-c7704dce01c5 tempest-TestNetworkAdvancedServerOps-1005354344 tempest-TestNetworkAdvancedServerOps-1005354344-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:31:05.344714 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-46485b9d-d213-446e-8d95-c7704dce01c5 tempest-TestNetworkAdvancedServerOps-1005354344 tempest-TestNetworkAdvancedServerOps-1005354344-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:31:05.345135 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-46485b9d-d213-446e-8d95-c7704dce01c5 tempest-TestNetworkAdvancedServerOps-1005354344 tempest-TestNetworkAdvancedServerOps-1005354344-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:31:05.361148 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-46485b9d-d213-446e-8d95-c7704dce01c5 tempest-TestNetworkAdvancedServerOps-1005354344 tempest-TestNetworkAdvancedServerOps-1005354344-project] Getting quotas for project 76e04ea1eac74b8eaf7062d92184c480. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 18 23:31:05.366223 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-46485b9d-d213-446e-8d95-c7704dce01c5 tempest-TestNetworkAdvancedServerOps-1005354344 tempest-TestNetworkAdvancedServerOps-1005354344-project] Getting quotas for user 46860dd540c9442e904c6404a08140c4 and project 76e04ea1eac74b8eaf7062d92184c480. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 18 23:31:05.377629 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-46485b9d-d213-446e-8d95-c7704dce01c5 tempest-TestNetworkAdvancedServerOps-1005354344 tempest-TestNetworkAdvancedServerOps-1005354344-project] [instance: 33748856-1973-44fc-ac52-b4b233f360d1] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120657) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 18 23:31:05.378347 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-46485b9d-d213-446e-8d95-c7704dce01c5 tempest-TestNetworkAdvancedServerOps-1005354344 tempest-TestNetworkAdvancedServerOps-1005354344-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:31:05.378789 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-46485b9d-d213-446e-8d95-c7704dce01c5 tempest-TestNetworkAdvancedServerOps-1005354344 tempest-TestNetworkAdvancedServerOps-1005354344-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:31:05.379180 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-46485b9d-d213-446e-8d95-c7704dce01c5 tempest-TestNetworkAdvancedServerOps-1005354344 tempest-TestNetworkAdvancedServerOps-1005354344-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:31:05.449988 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-46485b9d-d213-446e-8d95-c7704dce01c5 tempest-TestNetworkAdvancedServerOps-1005354344 tempest-TestNetworkAdvancedServerOps-1005354344-project] [instance: 33748856-1973-44fc-ac52-b4b233f360d1] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='87b0fec9-a948-41f9-a3c0-f0a6a5a8f1e7',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120657) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 18 23:31:05.451215 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-46485b9d-d213-446e-8d95-c7704dce01c5 tempest-TestNetworkAdvancedServerOps-1005354344 tempest-TestNetworkAdvancedServerOps-1005354344-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:31:05.451730 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-46485b9d-d213-446e-8d95-c7704dce01c5 tempest-TestNetworkAdvancedServerOps-1005354344 tempest-TestNetworkAdvancedServerOps-1005354344-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:31:05.452080 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-46485b9d-d213-446e-8d95-c7704dce01c5 tempest-TestNetworkAdvancedServerOps-1005354344 tempest-TestNetworkAdvancedServerOps-1005354344-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:31:05.753691 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-46485b9d-d213-446e-8d95-c7704dce01c5 tempest-TestNetworkAdvancedServerOps-1005354344 tempest-TestNetworkAdvancedServerOps-1005354344-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:31:05.754364 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-46485b9d-d213-446e-8d95-c7704dce01c5 tempest-TestNetworkAdvancedServerOps-1005354344 tempest-TestNetworkAdvancedServerOps-1005354344-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:31:05.754957 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-46485b9d-d213-446e-8d95-c7704dce01c5 tempest-TestNetworkAdvancedServerOps-1005354344 tempest-TestNetworkAdvancedServerOps-1005354344-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:31:05.755568 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-46485b9d-d213-446e-8d95-c7704dce01c5 tempest-TestNetworkAdvancedServerOps-1005354344 tempest-TestNetworkAdvancedServerOps-1005354344-project] Acquiring lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 18 23:31:05.756099 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-46485b9d-d213-446e-8d95-c7704dce01c5 tempest-TestNetworkAdvancedServerOps-1005354344 tempest-TestNetworkAdvancedServerOps-1005354344-project] Acquired lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 18 23:31:05.757913 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-ba0fdb30-5d1f-4f7c-ae26-3e93422709c5 None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:31:05.758700 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-ba0fdb30-5d1f-4f7c-ae26-3e93422709c5 None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:31:05.759396 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-ba0fdb30-5d1f-4f7c-ae26-3e93422709c5 None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:31:05.761906 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-ba0fdb30-5d1f-4f7c-ae26-3e93422709c5 None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:31:05.762352 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-ba0fdb30-5d1f-4f7c-ae26-3e93422709c5 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:31:05.762698 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-ba0fdb30-5d1f-4f7c-ae26-3e93422709c5 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:31:05.775559 n-d-853704-1 nova-conductor[120657]: ERROR nova.compute.rpcapi [None req-ba0fdb30-5d1f-4f7c-ae26-3e93422709c5 None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 18 23:31:05.777744 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-ba0fdb30-5d1f-4f7c-ae26-3e93422709c5 None None] Releasing lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 18 23:31:05.778697 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server [None req-ba0fdb30-5d1f-4f7c-ae26-3e93422709c5 None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 23:31:05.778697 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 23:31:05.778697 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 18 23:31:05.778697 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 18 23:31:05.778697 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 18 23:31:05.778697 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 23:31:05.778697 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 18 23:31:05.778697 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 23:31:05.778697 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 23:31:05.778697 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 18 23:31:05.778697 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 18 23:31:05.778697 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 18 23:31:05.778697 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 18 23:31:05.778697 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 18 23:31:05.778697 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 18 23:31:05.778697 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 18 23:31:05.778697 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 18 23:31:05.778697 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 18 23:31:05.778697 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 18 23:31:05.778697 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 18 23:31:05.778697 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 18 23:31:05.778697 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 18 23:31:05.778697 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 18 23:31:05.778697 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 23:31:05.778697 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 23:31:13.400980 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=12,nova_cell0:UPDATE=3 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:31:15.664305 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=15 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:31:15.665368 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=9,nova_api:UPDATE=1,nova_api:DELETE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:31:15.771252 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:31:15.772623 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell1:SELECT=11,nova_cell1:SAVEPOINT=1,nova_cell1:INSERT=18,nova_cell1:RELEASE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:31:23.407394 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=15,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:31:53.393313 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:32:03.405187 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:32:33.387615 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:32:43.414318 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=7,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:33:13.390173 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:33:21.340612 n-d-853704-1 nova-conductor[120658]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 18 23:33:23.412544 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:33:35.396856 n-d-853704-1 nova-conductor[120657]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 18 23:33:53.392982 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:34:03.419653 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=7,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:34:33.396128 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:34:43.416497 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:35:03.422421 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=3,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:35:13.398276 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:35:33.400464 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=3,nova_cell0:UPDATE=2 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:35:33.422122 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=6,nova_cell0:UPDATE=3 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:35:34.546477 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-191a1f89-066b-4e0a-83f4-aafcaa01579f tempest-AttachVolumeNegativeTest-144902305 tempest-AttachVolumeNegativeTest-144902305-project] Took 1.55 seconds to select destinations for 1 instance(s). {{(pid=120658) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 18 23:35:34.566798 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-191a1f89-066b-4e0a-83f4-aafcaa01579f tempest-AttachVolumeNegativeTest-144902305 tempest-AttachVolumeNegativeTest-144902305-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:35:34.567231 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-191a1f89-066b-4e0a-83f4-aafcaa01579f tempest-AttachVolumeNegativeTest-144902305 tempest-AttachVolumeNegativeTest-144902305-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:35:34.567615 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-191a1f89-066b-4e0a-83f4-aafcaa01579f tempest-AttachVolumeNegativeTest-144902305 tempest-AttachVolumeNegativeTest-144902305-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:35:34.696183 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-191a1f89-066b-4e0a-83f4-aafcaa01579f tempest-AttachVolumeNegativeTest-144902305 tempest-AttachVolumeNegativeTest-144902305-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:35:34.696641 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-191a1f89-066b-4e0a-83f4-aafcaa01579f tempest-AttachVolumeNegativeTest-144902305 tempest-AttachVolumeNegativeTest-144902305-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:35:34.697678 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-191a1f89-066b-4e0a-83f4-aafcaa01579f tempest-AttachVolumeNegativeTest-144902305 tempest-AttachVolumeNegativeTest-144902305-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:35:34.698344 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-191a1f89-066b-4e0a-83f4-aafcaa01579f tempest-AttachVolumeNegativeTest-144902305 tempest-AttachVolumeNegativeTest-144902305-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:35:34.698750 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-191a1f89-066b-4e0a-83f4-aafcaa01579f tempest-AttachVolumeNegativeTest-144902305 tempest-AttachVolumeNegativeTest-144902305-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:35:34.699133 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-191a1f89-066b-4e0a-83f4-aafcaa01579f tempest-AttachVolumeNegativeTest-144902305 tempest-AttachVolumeNegativeTest-144902305-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:35:34.715035 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-191a1f89-066b-4e0a-83f4-aafcaa01579f tempest-AttachVolumeNegativeTest-144902305 tempest-AttachVolumeNegativeTest-144902305-project] Getting quotas for project 666550d4121145498c79960dde4a67a0. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 18 23:35:34.719870 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-191a1f89-066b-4e0a-83f4-aafcaa01579f tempest-AttachVolumeNegativeTest-144902305 tempest-AttachVolumeNegativeTest-144902305-project] Getting quotas for user adf3f0e43b41460797d29d4f20438f8f and project 666550d4121145498c79960dde4a67a0. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 18 23:35:34.730253 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-191a1f89-066b-4e0a-83f4-aafcaa01579f tempest-AttachVolumeNegativeTest-144902305 tempest-AttachVolumeNegativeTest-144902305-project] [instance: 8de70eb8-ee30-493c-85e5-a5b61d2a2dc9] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120658) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 18 23:35:34.731020 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-191a1f89-066b-4e0a-83f4-aafcaa01579f tempest-AttachVolumeNegativeTest-144902305 tempest-AttachVolumeNegativeTest-144902305-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:35:34.731451 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-191a1f89-066b-4e0a-83f4-aafcaa01579f tempest-AttachVolumeNegativeTest-144902305 tempest-AttachVolumeNegativeTest-144902305-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:35:34.731867 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-191a1f89-066b-4e0a-83f4-aafcaa01579f tempest-AttachVolumeNegativeTest-144902305 tempest-AttachVolumeNegativeTest-144902305-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:35:34.822423 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-191a1f89-066b-4e0a-83f4-aafcaa01579f tempest-AttachVolumeNegativeTest-144902305 tempest-AttachVolumeNegativeTest-144902305-project] [instance: 8de70eb8-ee30-493c-85e5-a5b61d2a2dc9] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='87b0fec9-a948-41f9-a3c0-f0a6a5a8f1e7',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120658) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 18 23:35:34.823785 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-191a1f89-066b-4e0a-83f4-aafcaa01579f tempest-AttachVolumeNegativeTest-144902305 tempest-AttachVolumeNegativeTest-144902305-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:35:34.824219 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-191a1f89-066b-4e0a-83f4-aafcaa01579f tempest-AttachVolumeNegativeTest-144902305 tempest-AttachVolumeNegativeTest-144902305-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:35:34.824608 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-191a1f89-066b-4e0a-83f4-aafcaa01579f tempest-AttachVolumeNegativeTest-144902305 tempest-AttachVolumeNegativeTest-144902305-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:35:35.085658 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-191a1f89-066b-4e0a-83f4-aafcaa01579f tempest-AttachVolumeNegativeTest-144902305 tempest-AttachVolumeNegativeTest-144902305-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:35:35.086106 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-191a1f89-066b-4e0a-83f4-aafcaa01579f tempest-AttachVolumeNegativeTest-144902305 tempest-AttachVolumeNegativeTest-144902305-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:35:35.086757 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-191a1f89-066b-4e0a-83f4-aafcaa01579f tempest-AttachVolumeNegativeTest-144902305 tempest-AttachVolumeNegativeTest-144902305-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:35:35.087206 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-191a1f89-066b-4e0a-83f4-aafcaa01579f tempest-AttachVolumeNegativeTest-144902305 tempest-AttachVolumeNegativeTest-144902305-project] Acquiring lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 18 23:35:35.087572 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-191a1f89-066b-4e0a-83f4-aafcaa01579f tempest-AttachVolumeNegativeTest-144902305 tempest-AttachVolumeNegativeTest-144902305-project] Acquired lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 18 23:35:35.088637 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-bad5f136-b0a7-46bb-8365-bc48700fa537 None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:35:35.089046 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-bad5f136-b0a7-46bb-8365-bc48700fa537 None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:35:35.089428 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-bad5f136-b0a7-46bb-8365-bc48700fa537 None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:35:35.090072 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-bad5f136-b0a7-46bb-8365-bc48700fa537 None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:35:35.090478 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-bad5f136-b0a7-46bb-8365-bc48700fa537 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:35:35.090853 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-bad5f136-b0a7-46bb-8365-bc48700fa537 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:35:35.099178 n-d-853704-1 nova-conductor[120658]: ERROR nova.compute.rpcapi [None req-bad5f136-b0a7-46bb-8365-bc48700fa537 None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 18 23:35:35.099564 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-bad5f136-b0a7-46bb-8365-bc48700fa537 None None] Releasing lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 18 23:35:35.100596 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server [None req-bad5f136-b0a7-46bb-8365-bc48700fa537 None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 23:35:35.100596 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 23:35:35.100596 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 18 23:35:35.100596 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 18 23:35:35.100596 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 18 23:35:35.100596 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 23:35:35.100596 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 18 23:35:35.100596 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 23:35:35.100596 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 23:35:35.100596 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 18 23:35:35.100596 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 18 23:35:35.100596 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 18 23:35:35.100596 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 18 23:35:35.100596 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 18 23:35:35.100596 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 18 23:35:35.100596 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 18 23:35:35.100596 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 18 23:35:35.100596 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 18 23:35:35.100596 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 18 23:35:35.100596 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 18 23:35:35.100596 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 18 23:35:35.100596 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 18 23:35:35.100596 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 18 23:35:35.100596 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 23:35:35.100596 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 23:35:45.012910 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=15 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:35:45.013763 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=9,nova_api:UPDATE=1,nova_api:DELETE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:35:45.096620 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:35:45.098796 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell1:SELECT=11,nova_cell1:INSERT=17 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:35:53.402615 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=10,nova_cell0:UPDATE=2 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:35:53.424200 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=4,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:36:13.050402 n-d-853704-1 nova-conductor[120657]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 18 23:36:13.412274 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=4,nova_cell0:UPDATE=2 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:36:13.435055 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=4,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:36:53.412150 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:36:53.438594 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:36:55.405313 n-d-853704-1 nova-conductor[120657]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 18 23:37:21.139797 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-8fb39d86-51e0-4f6f-9d6f-f1595b738213 tempest-TestNetworkAdvancedServerOps-1005354344 tempest-TestNetworkAdvancedServerOps-1005354344-project] Took 0.60 seconds to select destinations for 1 instance(s). {{(pid=120657) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 18 23:37:21.161095 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-8fb39d86-51e0-4f6f-9d6f-f1595b738213 tempest-TestNetworkAdvancedServerOps-1005354344 tempest-TestNetworkAdvancedServerOps-1005354344-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:37:21.161621 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-8fb39d86-51e0-4f6f-9d6f-f1595b738213 tempest-TestNetworkAdvancedServerOps-1005354344 tempest-TestNetworkAdvancedServerOps-1005354344-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:37:21.162115 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-8fb39d86-51e0-4f6f-9d6f-f1595b738213 tempest-TestNetworkAdvancedServerOps-1005354344 tempest-TestNetworkAdvancedServerOps-1005354344-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:37:21.334981 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-8fb39d86-51e0-4f6f-9d6f-f1595b738213 tempest-TestNetworkAdvancedServerOps-1005354344 tempest-TestNetworkAdvancedServerOps-1005354344-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:37:21.335440 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-8fb39d86-51e0-4f6f-9d6f-f1595b738213 tempest-TestNetworkAdvancedServerOps-1005354344 tempest-TestNetworkAdvancedServerOps-1005354344-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:37:21.335917 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-8fb39d86-51e0-4f6f-9d6f-f1595b738213 tempest-TestNetworkAdvancedServerOps-1005354344 tempest-TestNetworkAdvancedServerOps-1005354344-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:37:21.336575 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-8fb39d86-51e0-4f6f-9d6f-f1595b738213 tempest-TestNetworkAdvancedServerOps-1005354344 tempest-TestNetworkAdvancedServerOps-1005354344-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:37:21.337646 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-8fb39d86-51e0-4f6f-9d6f-f1595b738213 tempest-TestNetworkAdvancedServerOps-1005354344 tempest-TestNetworkAdvancedServerOps-1005354344-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:37:21.338704 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-8fb39d86-51e0-4f6f-9d6f-f1595b738213 tempest-TestNetworkAdvancedServerOps-1005354344 tempest-TestNetworkAdvancedServerOps-1005354344-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:37:21.357072 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-8fb39d86-51e0-4f6f-9d6f-f1595b738213 tempest-TestNetworkAdvancedServerOps-1005354344 tempest-TestNetworkAdvancedServerOps-1005354344-project] Getting quotas for project 76e04ea1eac74b8eaf7062d92184c480. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 18 23:37:21.361792 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-8fb39d86-51e0-4f6f-9d6f-f1595b738213 tempest-TestNetworkAdvancedServerOps-1005354344 tempest-TestNetworkAdvancedServerOps-1005354344-project] Getting quotas for user 46860dd540c9442e904c6404a08140c4 and project 76e04ea1eac74b8eaf7062d92184c480. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 18 23:37:21.373931 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-8fb39d86-51e0-4f6f-9d6f-f1595b738213 tempest-TestNetworkAdvancedServerOps-1005354344 tempest-TestNetworkAdvancedServerOps-1005354344-project] [instance: 00cf860c-0fd2-4d02-ab58-cb9a4deec831] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120657) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 18 23:37:21.374891 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-8fb39d86-51e0-4f6f-9d6f-f1595b738213 tempest-TestNetworkAdvancedServerOps-1005354344 tempest-TestNetworkAdvancedServerOps-1005354344-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:37:21.375335 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-8fb39d86-51e0-4f6f-9d6f-f1595b738213 tempest-TestNetworkAdvancedServerOps-1005354344 tempest-TestNetworkAdvancedServerOps-1005354344-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:37:21.375861 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-8fb39d86-51e0-4f6f-9d6f-f1595b738213 tempest-TestNetworkAdvancedServerOps-1005354344 tempest-TestNetworkAdvancedServerOps-1005354344-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:37:21.488698 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-8fb39d86-51e0-4f6f-9d6f-f1595b738213 tempest-TestNetworkAdvancedServerOps-1005354344 tempest-TestNetworkAdvancedServerOps-1005354344-project] [instance: 00cf860c-0fd2-4d02-ab58-cb9a4deec831] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='87b0fec9-a948-41f9-a3c0-f0a6a5a8f1e7',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120657) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 18 23:37:21.489951 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-8fb39d86-51e0-4f6f-9d6f-f1595b738213 tempest-TestNetworkAdvancedServerOps-1005354344 tempest-TestNetworkAdvancedServerOps-1005354344-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:37:21.490392 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-8fb39d86-51e0-4f6f-9d6f-f1595b738213 tempest-TestNetworkAdvancedServerOps-1005354344 tempest-TestNetworkAdvancedServerOps-1005354344-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:37:21.490785 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-8fb39d86-51e0-4f6f-9d6f-f1595b738213 tempest-TestNetworkAdvancedServerOps-1005354344 tempest-TestNetworkAdvancedServerOps-1005354344-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:37:21.800747 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-8fb39d86-51e0-4f6f-9d6f-f1595b738213 tempest-TestNetworkAdvancedServerOps-1005354344 tempest-TestNetworkAdvancedServerOps-1005354344-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:37:21.801191 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-8fb39d86-51e0-4f6f-9d6f-f1595b738213 tempest-TestNetworkAdvancedServerOps-1005354344 tempest-TestNetworkAdvancedServerOps-1005354344-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:37:21.801583 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-8fb39d86-51e0-4f6f-9d6f-f1595b738213 tempest-TestNetworkAdvancedServerOps-1005354344 tempest-TestNetworkAdvancedServerOps-1005354344-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:37:21.802035 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-8fb39d86-51e0-4f6f-9d6f-f1595b738213 tempest-TestNetworkAdvancedServerOps-1005354344 tempest-TestNetworkAdvancedServerOps-1005354344-project] Acquiring lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 18 23:37:21.802426 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-8fb39d86-51e0-4f6f-9d6f-f1595b738213 tempest-TestNetworkAdvancedServerOps-1005354344 tempest-TestNetworkAdvancedServerOps-1005354344-project] Acquired lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 18 23:37:21.803150 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-8859d8e4-09a5-4ed4-8b7c-342e69b701ad None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:37:21.803559 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-8859d8e4-09a5-4ed4-8b7c-342e69b701ad None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:37:21.803986 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-8859d8e4-09a5-4ed4-8b7c-342e69b701ad None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:37:21.804658 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-8859d8e4-09a5-4ed4-8b7c-342e69b701ad None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:37:21.805084 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-8859d8e4-09a5-4ed4-8b7c-342e69b701ad None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:37:21.805461 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-8859d8e4-09a5-4ed4-8b7c-342e69b701ad None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:37:21.824773 n-d-853704-1 nova-conductor[120657]: ERROR nova.compute.rpcapi [None req-8859d8e4-09a5-4ed4-8b7c-342e69b701ad None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 18 23:37:21.825171 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-8859d8e4-09a5-4ed4-8b7c-342e69b701ad None None] Releasing lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 18 23:37:21.826104 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server [None req-8859d8e4-09a5-4ed4-8b7c-342e69b701ad None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 23:37:21.826104 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 23:37:21.826104 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 18 23:37:21.826104 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 18 23:37:21.826104 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 18 23:37:21.826104 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 23:37:21.826104 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 18 23:37:21.826104 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 23:37:21.826104 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 23:37:21.826104 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 18 23:37:21.826104 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 18 23:37:21.826104 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 18 23:37:21.826104 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 18 23:37:21.826104 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 18 23:37:21.826104 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 18 23:37:21.826104 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 18 23:37:21.826104 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 18 23:37:21.826104 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 18 23:37:21.826104 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 18 23:37:21.826104 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 18 23:37:21.826104 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 18 23:37:21.826104 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 18 23:37:21.826104 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 18 23:37:21.826104 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 23:37:21.826104 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 23:37:31.587318 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=11 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:37:31.689752 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=13,nova_api:UPDATE=1,nova_api:DELETE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:37:31.818373 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:37:31.819187 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell1:SELECT=11,nova_cell1:INSERT=17 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:37:33.401209 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:37:33.433191 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=14,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:37:53.402925 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=3,nova_cell0:UPDATE=2 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:37:55.280817 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-14b120db-0cbd-4f7b-a321-7e8d0a35933c tempest-VolumesNegativeTest-1032067711 tempest-VolumesNegativeTest-1032067711-project] Took 0.53 seconds to select destinations for 1 instance(s). {{(pid=120658) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 18 23:37:55.300432 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-14b120db-0cbd-4f7b-a321-7e8d0a35933c tempest-VolumesNegativeTest-1032067711 tempest-VolumesNegativeTest-1032067711-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:37:55.300878 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-14b120db-0cbd-4f7b-a321-7e8d0a35933c tempest-VolumesNegativeTest-1032067711 tempest-VolumesNegativeTest-1032067711-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:37:55.301265 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-14b120db-0cbd-4f7b-a321-7e8d0a35933c tempest-VolumesNegativeTest-1032067711 tempest-VolumesNegativeTest-1032067711-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:37:55.583577 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-14b120db-0cbd-4f7b-a321-7e8d0a35933c tempest-VolumesNegativeTest-1032067711 tempest-VolumesNegativeTest-1032067711-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:37:55.584072 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-14b120db-0cbd-4f7b-a321-7e8d0a35933c tempest-VolumesNegativeTest-1032067711 tempest-VolumesNegativeTest-1032067711-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:37:55.584728 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-14b120db-0cbd-4f7b-a321-7e8d0a35933c tempest-VolumesNegativeTest-1032067711 tempest-VolumesNegativeTest-1032067711-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:37:55.585391 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-14b120db-0cbd-4f7b-a321-7e8d0a35933c tempest-VolumesNegativeTest-1032067711 tempest-VolumesNegativeTest-1032067711-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:37:55.585862 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-14b120db-0cbd-4f7b-a321-7e8d0a35933c tempest-VolumesNegativeTest-1032067711 tempest-VolumesNegativeTest-1032067711-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:37:55.586261 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-14b120db-0cbd-4f7b-a321-7e8d0a35933c tempest-VolumesNegativeTest-1032067711 tempest-VolumesNegativeTest-1032067711-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:37:55.602724 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-14b120db-0cbd-4f7b-a321-7e8d0a35933c tempest-VolumesNegativeTest-1032067711 tempest-VolumesNegativeTest-1032067711-project] Getting quotas for project 53c2bfdedbcb4c989545493e0273843e. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 18 23:37:55.608111 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-14b120db-0cbd-4f7b-a321-7e8d0a35933c tempest-VolumesNegativeTest-1032067711 tempest-VolumesNegativeTest-1032067711-project] Getting quotas for user 3ef58357c6e44dd7b8dadfa0faf0137d and project 53c2bfdedbcb4c989545493e0273843e. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 18 23:37:55.620423 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-14b120db-0cbd-4f7b-a321-7e8d0a35933c tempest-VolumesNegativeTest-1032067711 tempest-VolumesNegativeTest-1032067711-project] [instance: 382b0f20-d038-469a-80da-410356513fa0] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120658) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 18 23:37:55.621669 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-14b120db-0cbd-4f7b-a321-7e8d0a35933c tempest-VolumesNegativeTest-1032067711 tempest-VolumesNegativeTest-1032067711-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:37:55.621775 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-14b120db-0cbd-4f7b-a321-7e8d0a35933c tempest-VolumesNegativeTest-1032067711 tempest-VolumesNegativeTest-1032067711-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:37:55.622126 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-14b120db-0cbd-4f7b-a321-7e8d0a35933c tempest-VolumesNegativeTest-1032067711 tempest-VolumesNegativeTest-1032067711-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:37:55.785355 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-14b120db-0cbd-4f7b-a321-7e8d0a35933c tempest-VolumesNegativeTest-1032067711 tempest-VolumesNegativeTest-1032067711-project] [instance: 382b0f20-d038-469a-80da-410356513fa0] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='87b0fec9-a948-41f9-a3c0-f0a6a5a8f1e7',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120658) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 18 23:37:55.786809 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-14b120db-0cbd-4f7b-a321-7e8d0a35933c tempest-VolumesNegativeTest-1032067711 tempest-VolumesNegativeTest-1032067711-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:37:55.787316 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-14b120db-0cbd-4f7b-a321-7e8d0a35933c tempest-VolumesNegativeTest-1032067711 tempest-VolumesNegativeTest-1032067711-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:37:55.787980 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-14b120db-0cbd-4f7b-a321-7e8d0a35933c tempest-VolumesNegativeTest-1032067711 tempest-VolumesNegativeTest-1032067711-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:37:56.003125 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-14b120db-0cbd-4f7b-a321-7e8d0a35933c tempest-VolumesNegativeTest-1032067711 tempest-VolumesNegativeTest-1032067711-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:37:56.003547 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-14b120db-0cbd-4f7b-a321-7e8d0a35933c tempest-VolumesNegativeTest-1032067711 tempest-VolumesNegativeTest-1032067711-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:37:56.003971 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-14b120db-0cbd-4f7b-a321-7e8d0a35933c tempest-VolumesNegativeTest-1032067711 tempest-VolumesNegativeTest-1032067711-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:37:56.004415 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-14b120db-0cbd-4f7b-a321-7e8d0a35933c tempest-VolumesNegativeTest-1032067711 tempest-VolumesNegativeTest-1032067711-project] Acquiring lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 18 23:37:56.004788 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-14b120db-0cbd-4f7b-a321-7e8d0a35933c tempest-VolumesNegativeTest-1032067711 tempest-VolumesNegativeTest-1032067711-project] Acquired lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 18 23:37:56.005509 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-a93147d2-99fb-4c40-b09d-0207dc4d73e2 None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:37:56.005916 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-a93147d2-99fb-4c40-b09d-0207dc4d73e2 None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:37:56.006485 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-a93147d2-99fb-4c40-b09d-0207dc4d73e2 None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:37:56.007128 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-a93147d2-99fb-4c40-b09d-0207dc4d73e2 None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:37:56.007533 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-a93147d2-99fb-4c40-b09d-0207dc4d73e2 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:37:56.007939 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-a93147d2-99fb-4c40-b09d-0207dc4d73e2 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:37:56.017008 n-d-853704-1 nova-conductor[120658]: ERROR nova.compute.rpcapi [None req-a93147d2-99fb-4c40-b09d-0207dc4d73e2 None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 18 23:37:56.017864 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-a93147d2-99fb-4c40-b09d-0207dc4d73e2 None None] Releasing lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 18 23:37:56.018370 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server [None req-a93147d2-99fb-4c40-b09d-0207dc4d73e2 None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 23:37:56.018370 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 23:37:56.018370 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 18 23:37:56.018370 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 18 23:37:56.018370 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 18 23:37:56.018370 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 23:37:56.018370 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 18 23:37:56.018370 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 23:37:56.018370 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 23:37:56.018370 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 18 23:37:56.018370 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 18 23:37:56.018370 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 18 23:37:56.018370 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 18 23:37:56.018370 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 18 23:37:56.018370 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 18 23:37:56.018370 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 18 23:37:56.018370 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 18 23:37:56.018370 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 18 23:37:56.018370 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 18 23:37:56.018370 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 18 23:37:56.018370 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 18 23:37:56.018370 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 18 23:37:56.018370 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 18 23:37:56.018370 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 23:37:56.018370 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 23:38:05.952825 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=11 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:38:05.954795 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=13,nova_api:UPDATE=1,nova_api:DELETE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:38:06.014778 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:38:06.016107 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell1:SELECT=11,nova_cell1:SAVEPOINT=1,nova_cell1:INSERT=18,nova_cell1:RELEASE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:38:13.111799 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-e18a2bd1-fd37-42df-8573-d64c149c1031 tempest-TestVolumeBootPattern-1034437477 tempest-TestVolumeBootPattern-1034437477-project] Took 0.34 seconds to select destinations for 1 instance(s). {{(pid=120657) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 18 23:38:13.132798 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-e18a2bd1-fd37-42df-8573-d64c149c1031 tempest-TestVolumeBootPattern-1034437477 tempest-TestVolumeBootPattern-1034437477-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:38:13.133242 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-e18a2bd1-fd37-42df-8573-d64c149c1031 tempest-TestVolumeBootPattern-1034437477 tempest-TestVolumeBootPattern-1034437477-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:38:13.133636 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-e18a2bd1-fd37-42df-8573-d64c149c1031 tempest-TestVolumeBootPattern-1034437477 tempest-TestVolumeBootPattern-1034437477-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:38:13.247906 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:38:13.251073 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-e18a2bd1-fd37-42df-8573-d64c149c1031 tempest-TestVolumeBootPattern-1034437477 tempest-TestVolumeBootPattern-1034437477-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:38:13.251504 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-e18a2bd1-fd37-42df-8573-d64c149c1031 tempest-TestVolumeBootPattern-1034437477 tempest-TestVolumeBootPattern-1034437477-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:38:13.251918 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-e18a2bd1-fd37-42df-8573-d64c149c1031 tempest-TestVolumeBootPattern-1034437477 tempest-TestVolumeBootPattern-1034437477-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:38:13.254508 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-e18a2bd1-fd37-42df-8573-d64c149c1031 tempest-TestVolumeBootPattern-1034437477 tempest-TestVolumeBootPattern-1034437477-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:38:13.254508 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-e18a2bd1-fd37-42df-8573-d64c149c1031 tempest-TestVolumeBootPattern-1034437477 tempest-TestVolumeBootPattern-1034437477-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:38:13.254508 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-e18a2bd1-fd37-42df-8573-d64c149c1031 tempest-TestVolumeBootPattern-1034437477 tempest-TestVolumeBootPattern-1034437477-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:38:13.269030 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-e18a2bd1-fd37-42df-8573-d64c149c1031 tempest-TestVolumeBootPattern-1034437477 tempest-TestVolumeBootPattern-1034437477-project] Getting quotas for project dfab7231c21e4b1f82cb2fbf1c4e0986. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 18 23:38:13.273680 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-e18a2bd1-fd37-42df-8573-d64c149c1031 tempest-TestVolumeBootPattern-1034437477 tempest-TestVolumeBootPattern-1034437477-project] Getting quotas for user 480cd2375ac24f8a89b9730fcaa6965e and project dfab7231c21e4b1f82cb2fbf1c4e0986. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 18 23:38:13.285328 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-e18a2bd1-fd37-42df-8573-d64c149c1031 tempest-TestVolumeBootPattern-1034437477 tempest-TestVolumeBootPattern-1034437477-project] [instance: d79f34af-f411-4389-8f7b-54525c8c61f4] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120657) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 18 23:38:13.286077 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-e18a2bd1-fd37-42df-8573-d64c149c1031 tempest-TestVolumeBootPattern-1034437477 tempest-TestVolumeBootPattern-1034437477-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:38:13.286634 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-e18a2bd1-fd37-42df-8573-d64c149c1031 tempest-TestVolumeBootPattern-1034437477 tempest-TestVolumeBootPattern-1034437477-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:38:13.287029 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-e18a2bd1-fd37-42df-8573-d64c149c1031 tempest-TestVolumeBootPattern-1034437477 tempest-TestVolumeBootPattern-1034437477-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:38:13.348105 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-e18a2bd1-fd37-42df-8573-d64c149c1031 tempest-TestVolumeBootPattern-1034437477 tempest-TestVolumeBootPattern-1034437477-project] [instance: d79f34af-f411-4389-8f7b-54525c8c61f4] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='volume',device_name=None,device_type=None,disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id=None,instance=,instance_uuid=,no_device=False,snapshot_id='8d961d60-4b17-479a-8ccc-1113165fb2e3',source_type='snapshot',tag=None,updated_at=,uuid=,volume_id=None,volume_size=1,volume_type=None)] {{(pid=120657) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 18 23:38:13.349640 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-e18a2bd1-fd37-42df-8573-d64c149c1031 tempest-TestVolumeBootPattern-1034437477 tempest-TestVolumeBootPattern-1034437477-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:38:13.350081 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-e18a2bd1-fd37-42df-8573-d64c149c1031 tempest-TestVolumeBootPattern-1034437477 tempest-TestVolumeBootPattern-1034437477-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:38:13.350473 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-e18a2bd1-fd37-42df-8573-d64c149c1031 tempest-TestVolumeBootPattern-1034437477 tempest-TestVolumeBootPattern-1034437477-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:38:13.608505 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-e18a2bd1-fd37-42df-8573-d64c149c1031 tempest-TestVolumeBootPattern-1034437477 tempest-TestVolumeBootPattern-1034437477-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:38:13.608943 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-e18a2bd1-fd37-42df-8573-d64c149c1031 tempest-TestVolumeBootPattern-1034437477 tempest-TestVolumeBootPattern-1034437477-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:38:13.609330 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-e18a2bd1-fd37-42df-8573-d64c149c1031 tempest-TestVolumeBootPattern-1034437477 tempest-TestVolumeBootPattern-1034437477-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:38:13.609779 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-e18a2bd1-fd37-42df-8573-d64c149c1031 tempest-TestVolumeBootPattern-1034437477 tempest-TestVolumeBootPattern-1034437477-project] Acquiring lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 18 23:38:13.610127 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-e18a2bd1-fd37-42df-8573-d64c149c1031 tempest-TestVolumeBootPattern-1034437477 tempest-TestVolumeBootPattern-1034437477-project] Acquired lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 18 23:38:13.610844 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-ad8b92de-b636-4f28-b72f-389f79d3e956 None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:38:13.611250 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-ad8b92de-b636-4f28-b72f-389f79d3e956 None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:38:13.611625 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-ad8b92de-b636-4f28-b72f-389f79d3e956 None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:38:13.612289 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-ad8b92de-b636-4f28-b72f-389f79d3e956 None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:38:13.612840 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-ad8b92de-b636-4f28-b72f-389f79d3e956 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:38:13.613425 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-ad8b92de-b636-4f28-b72f-389f79d3e956 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:38:13.621900 n-d-853704-1 nova-conductor[120657]: ERROR nova.compute.rpcapi [None req-ad8b92de-b636-4f28-b72f-389f79d3e956 None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 18 23:38:13.622281 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-ad8b92de-b636-4f28-b72f-389f79d3e956 None None] Releasing lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 18 23:38:13.623370 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server [None req-ad8b92de-b636-4f28-b72f-389f79d3e956 None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 23:38:13.623370 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 23:38:13.623370 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 18 23:38:13.623370 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 18 23:38:13.623370 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 18 23:38:13.623370 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 23:38:13.623370 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 18 23:38:13.623370 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 23:38:13.623370 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 23:38:13.623370 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 18 23:38:13.623370 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 18 23:38:13.623370 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 18 23:38:13.623370 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 18 23:38:13.623370 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 18 23:38:13.623370 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 18 23:38:13.623370 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 18 23:38:13.623370 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 18 23:38:13.623370 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 18 23:38:13.623370 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 18 23:38:13.623370 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 18 23:38:13.623370 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 18 23:38:13.623370 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 18 23:38:13.623370 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 18 23:38:13.623370 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 23:38:13.623370 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 23:38:23.448914 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=12 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:38:23.550663 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=12,nova_api:UPDATE=1,nova_api:DELETE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:38:23.619747 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:38:23.620425 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell1:SELECT=11,nova_cell1:INSERT=16 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:38:33.403195 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=15,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:38:43.436850 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=11,nova_cell0:UPDATE=3 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:38:53.410451 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=3,nova_cell0:UPDATE=2 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:39:23.417924 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=6,nova_cell0:UPDATE=3 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:39:23.450047 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:40:03.417308 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:40:03.448851 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:40:15.414402 n-d-853704-1 nova-conductor[120657]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 18 23:40:43.404924 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:40:43.435363 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:41:03.439973 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=3,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:41:13.060155 n-d-853704-1 nova-conductor[120657]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 18 23:41:23.414213 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:41:23.445821 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=4,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:42:03.410386 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:42:03.440351 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:42:23.419092 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=3,nova_cell0:UPDATE=2 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:42:23.449110 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=3,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:43:03.428801 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:43:03.458858 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:43:21.355650 n-d-853704-1 nova-conductor[120658]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 18 23:43:23.433868 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=3,nova_cell0:UPDATE=2 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:43:35.423580 n-d-853704-1 nova-conductor[120657]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 18 23:43:39.617215 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-95335dc7-2704-4835-84a9-b9cab5d2f1fe tempest-TestNetworkAdvancedServerOps-1005354344 tempest-TestNetworkAdvancedServerOps-1005354344-project] Took 0.63 seconds to select destinations for 1 instance(s). {{(pid=120658) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 18 23:43:39.640052 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-95335dc7-2704-4835-84a9-b9cab5d2f1fe tempest-TestNetworkAdvancedServerOps-1005354344 tempest-TestNetworkAdvancedServerOps-1005354344-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:43:39.640738 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-95335dc7-2704-4835-84a9-b9cab5d2f1fe tempest-TestNetworkAdvancedServerOps-1005354344 tempest-TestNetworkAdvancedServerOps-1005354344-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:43:39.641139 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-95335dc7-2704-4835-84a9-b9cab5d2f1fe tempest-TestNetworkAdvancedServerOps-1005354344 tempest-TestNetworkAdvancedServerOps-1005354344-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:43:39.816706 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-95335dc7-2704-4835-84a9-b9cab5d2f1fe tempest-TestNetworkAdvancedServerOps-1005354344 tempest-TestNetworkAdvancedServerOps-1005354344-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:43:39.817070 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-95335dc7-2704-4835-84a9-b9cab5d2f1fe tempest-TestNetworkAdvancedServerOps-1005354344 tempest-TestNetworkAdvancedServerOps-1005354344-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:43:39.817532 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-95335dc7-2704-4835-84a9-b9cab5d2f1fe tempest-TestNetworkAdvancedServerOps-1005354344 tempest-TestNetworkAdvancedServerOps-1005354344-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:43:39.818397 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-95335dc7-2704-4835-84a9-b9cab5d2f1fe tempest-TestNetworkAdvancedServerOps-1005354344 tempest-TestNetworkAdvancedServerOps-1005354344-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:43:39.818938 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-95335dc7-2704-4835-84a9-b9cab5d2f1fe tempest-TestNetworkAdvancedServerOps-1005354344 tempest-TestNetworkAdvancedServerOps-1005354344-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:43:39.819550 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-95335dc7-2704-4835-84a9-b9cab5d2f1fe tempest-TestNetworkAdvancedServerOps-1005354344 tempest-TestNetworkAdvancedServerOps-1005354344-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:43:39.836817 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-95335dc7-2704-4835-84a9-b9cab5d2f1fe tempest-TestNetworkAdvancedServerOps-1005354344 tempest-TestNetworkAdvancedServerOps-1005354344-project] Getting quotas for project 76e04ea1eac74b8eaf7062d92184c480. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 18 23:43:39.841512 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-95335dc7-2704-4835-84a9-b9cab5d2f1fe tempest-TestNetworkAdvancedServerOps-1005354344 tempest-TestNetworkAdvancedServerOps-1005354344-project] Getting quotas for user 46860dd540c9442e904c6404a08140c4 and project 76e04ea1eac74b8eaf7062d92184c480. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 18 23:43:39.854547 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-95335dc7-2704-4835-84a9-b9cab5d2f1fe tempest-TestNetworkAdvancedServerOps-1005354344 tempest-TestNetworkAdvancedServerOps-1005354344-project] [instance: d9595790-4c64-4230-b536-a42712a973d6] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120658) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 18 23:43:39.855326 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-95335dc7-2704-4835-84a9-b9cab5d2f1fe tempest-TestNetworkAdvancedServerOps-1005354344 tempest-TestNetworkAdvancedServerOps-1005354344-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:43:39.855798 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-95335dc7-2704-4835-84a9-b9cab5d2f1fe tempest-TestNetworkAdvancedServerOps-1005354344 tempest-TestNetworkAdvancedServerOps-1005354344-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:43:39.856182 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-95335dc7-2704-4835-84a9-b9cab5d2f1fe tempest-TestNetworkAdvancedServerOps-1005354344 tempest-TestNetworkAdvancedServerOps-1005354344-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:43:39.927385 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-95335dc7-2704-4835-84a9-b9cab5d2f1fe tempest-TestNetworkAdvancedServerOps-1005354344 tempest-TestNetworkAdvancedServerOps-1005354344-project] [instance: d9595790-4c64-4230-b536-a42712a973d6] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='87b0fec9-a948-41f9-a3c0-f0a6a5a8f1e7',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120658) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 18 23:43:39.929064 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-95335dc7-2704-4835-84a9-b9cab5d2f1fe tempest-TestNetworkAdvancedServerOps-1005354344 tempest-TestNetworkAdvancedServerOps-1005354344-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:43:39.929699 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-95335dc7-2704-4835-84a9-b9cab5d2f1fe tempest-TestNetworkAdvancedServerOps-1005354344 tempest-TestNetworkAdvancedServerOps-1005354344-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:43:39.930089 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-95335dc7-2704-4835-84a9-b9cab5d2f1fe tempest-TestNetworkAdvancedServerOps-1005354344 tempest-TestNetworkAdvancedServerOps-1005354344-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:43:40.174414 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-95335dc7-2704-4835-84a9-b9cab5d2f1fe tempest-TestNetworkAdvancedServerOps-1005354344 tempest-TestNetworkAdvancedServerOps-1005354344-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:43:40.174742 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-95335dc7-2704-4835-84a9-b9cab5d2f1fe tempest-TestNetworkAdvancedServerOps-1005354344 tempest-TestNetworkAdvancedServerOps-1005354344-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:43:40.175592 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-95335dc7-2704-4835-84a9-b9cab5d2f1fe tempest-TestNetworkAdvancedServerOps-1005354344 tempest-TestNetworkAdvancedServerOps-1005354344-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:43:40.176954 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-95335dc7-2704-4835-84a9-b9cab5d2f1fe tempest-TestNetworkAdvancedServerOps-1005354344 tempest-TestNetworkAdvancedServerOps-1005354344-project] Acquiring lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 18 23:43:40.177517 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-95335dc7-2704-4835-84a9-b9cab5d2f1fe tempest-TestNetworkAdvancedServerOps-1005354344 tempest-TestNetworkAdvancedServerOps-1005354344-project] Acquired lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 18 23:43:40.180325 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-aa04e2ff-f84d-4bfa-96c1-e26c2e4f79e1 None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:43:40.184472 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-aa04e2ff-f84d-4bfa-96c1-e26c2e4f79e1 None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:43:40.184472 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-aa04e2ff-f84d-4bfa-96c1-e26c2e4f79e1 None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:43:40.184472 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-aa04e2ff-f84d-4bfa-96c1-e26c2e4f79e1 None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:43:40.184472 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-aa04e2ff-f84d-4bfa-96c1-e26c2e4f79e1 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:43:40.184472 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-aa04e2ff-f84d-4bfa-96c1-e26c2e4f79e1 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:43:40.193025 n-d-853704-1 nova-conductor[120658]: ERROR nova.compute.rpcapi [None req-aa04e2ff-f84d-4bfa-96c1-e26c2e4f79e1 None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 18 23:43:40.199798 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-aa04e2ff-f84d-4bfa-96c1-e26c2e4f79e1 None None] Releasing lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 18 23:43:40.200809 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server [None req-aa04e2ff-f84d-4bfa-96c1-e26c2e4f79e1 None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 23:43:40.200809 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 23:43:40.200809 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 18 23:43:40.200809 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 18 23:43:40.200809 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 18 23:43:40.200809 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 23:43:40.200809 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 18 23:43:40.200809 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 23:43:40.200809 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 23:43:40.200809 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 18 23:43:40.200809 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 18 23:43:40.200809 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 18 23:43:40.200809 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 18 23:43:40.200809 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 18 23:43:40.200809 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 18 23:43:40.200809 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 18 23:43:40.200809 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 18 23:43:40.200809 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 18 23:43:40.200809 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 18 23:43:40.200809 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 18 23:43:40.200809 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 18 23:43:40.200809 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 18 23:43:40.200809 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 18 23:43:40.200809 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 23:43:40.200809 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 23:43:43.469084 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:43:50.090334 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=12,nova_api:UPDATE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:43:50.091251 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=12,nova_api:DELETE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:43:50.192026 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:43:50.192684 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell1:SELECT=11,nova_cell1:INSERT=17 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:44:03.445132 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=15,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:44:10.631654 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-6e4b88d3-d6e5-4487-a886-00579798b6a8 tempest-TestServerAdvancedOps-937002030 tempest-TestServerAdvancedOps-937002030-project] Took 0.67 seconds to select destinations for 1 instance(s). {{(pid=120657) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 18 23:44:10.651360 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-6e4b88d3-d6e5-4487-a886-00579798b6a8 tempest-TestServerAdvancedOps-937002030 tempest-TestServerAdvancedOps-937002030-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:44:10.653583 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-6e4b88d3-d6e5-4487-a886-00579798b6a8 tempest-TestServerAdvancedOps-937002030 tempest-TestServerAdvancedOps-937002030-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:44:10.653583 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-6e4b88d3-d6e5-4487-a886-00579798b6a8 tempest-TestServerAdvancedOps-937002030 tempest-TestServerAdvancedOps-937002030-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:44:10.812690 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-6e4b88d3-d6e5-4487-a886-00579798b6a8 tempest-TestServerAdvancedOps-937002030 tempest-TestServerAdvancedOps-937002030-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:44:10.813133 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-6e4b88d3-d6e5-4487-a886-00579798b6a8 tempest-TestServerAdvancedOps-937002030 tempest-TestServerAdvancedOps-937002030-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:44:10.813547 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-6e4b88d3-d6e5-4487-a886-00579798b6a8 tempest-TestServerAdvancedOps-937002030 tempest-TestServerAdvancedOps-937002030-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:44:10.814173 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-6e4b88d3-d6e5-4487-a886-00579798b6a8 tempest-TestServerAdvancedOps-937002030 tempest-TestServerAdvancedOps-937002030-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:44:10.815063 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-6e4b88d3-d6e5-4487-a886-00579798b6a8 tempest-TestServerAdvancedOps-937002030 tempest-TestServerAdvancedOps-937002030-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:44:10.815448 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-6e4b88d3-d6e5-4487-a886-00579798b6a8 tempest-TestServerAdvancedOps-937002030 tempest-TestServerAdvancedOps-937002030-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:44:10.831254 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-6e4b88d3-d6e5-4487-a886-00579798b6a8 tempest-TestServerAdvancedOps-937002030 tempest-TestServerAdvancedOps-937002030-project] Getting quotas for project a8c91b7686fe4fbb9f016986bb58afaf. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 18 23:44:10.836632 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-6e4b88d3-d6e5-4487-a886-00579798b6a8 tempest-TestServerAdvancedOps-937002030 tempest-TestServerAdvancedOps-937002030-project] Getting quotas for user b6f0edbc98ba4074b24149fa3cacbbf5 and project a8c91b7686fe4fbb9f016986bb58afaf. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 18 23:44:10.847388 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-6e4b88d3-d6e5-4487-a886-00579798b6a8 tempest-TestServerAdvancedOps-937002030 tempest-TestServerAdvancedOps-937002030-project] [instance: 7b9081bb-05cc-4eb3-8792-0f2d7a366846] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120657) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 18 23:44:10.848199 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-6e4b88d3-d6e5-4487-a886-00579798b6a8 tempest-TestServerAdvancedOps-937002030 tempest-TestServerAdvancedOps-937002030-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:44:10.848633 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-6e4b88d3-d6e5-4487-a886-00579798b6a8 tempest-TestServerAdvancedOps-937002030 tempest-TestServerAdvancedOps-937002030-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:44:10.849755 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-6e4b88d3-d6e5-4487-a886-00579798b6a8 tempest-TestServerAdvancedOps-937002030 tempest-TestServerAdvancedOps-937002030-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:44:11.007251 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-6e4b88d3-d6e5-4487-a886-00579798b6a8 tempest-TestServerAdvancedOps-937002030 tempest-TestServerAdvancedOps-937002030-project] [instance: 7b9081bb-05cc-4eb3-8792-0f2d7a366846] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='87b0fec9-a948-41f9-a3c0-f0a6a5a8f1e7',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120657) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 18 23:44:11.008514 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-6e4b88d3-d6e5-4487-a886-00579798b6a8 tempest-TestServerAdvancedOps-937002030 tempest-TestServerAdvancedOps-937002030-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:44:11.008957 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-6e4b88d3-d6e5-4487-a886-00579798b6a8 tempest-TestServerAdvancedOps-937002030 tempest-TestServerAdvancedOps-937002030-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:44:11.009353 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-6e4b88d3-d6e5-4487-a886-00579798b6a8 tempest-TestServerAdvancedOps-937002030 tempest-TestServerAdvancedOps-937002030-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:44:11.269980 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-6e4b88d3-d6e5-4487-a886-00579798b6a8 tempest-TestServerAdvancedOps-937002030 tempest-TestServerAdvancedOps-937002030-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:44:11.270593 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-6e4b88d3-d6e5-4487-a886-00579798b6a8 tempest-TestServerAdvancedOps-937002030 tempest-TestServerAdvancedOps-937002030-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:44:11.271813 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-6e4b88d3-d6e5-4487-a886-00579798b6a8 tempest-TestServerAdvancedOps-937002030 tempest-TestServerAdvancedOps-937002030-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:44:11.275183 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-6e4b88d3-d6e5-4487-a886-00579798b6a8 tempest-TestServerAdvancedOps-937002030 tempest-TestServerAdvancedOps-937002030-project] Acquiring lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 18 23:44:11.275183 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-6e4b88d3-d6e5-4487-a886-00579798b6a8 tempest-TestServerAdvancedOps-937002030 tempest-TestServerAdvancedOps-937002030-project] Acquired lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 18 23:44:11.275183 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-a505e936-8dd0-4544-af0a-c60d1ade9956 None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:44:11.275183 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-a505e936-8dd0-4544-af0a-c60d1ade9956 None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:44:11.275183 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-a505e936-8dd0-4544-af0a-c60d1ade9956 None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:44:11.275183 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-a505e936-8dd0-4544-af0a-c60d1ade9956 None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:44:11.275814 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-a505e936-8dd0-4544-af0a-c60d1ade9956 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:44:11.275814 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-a505e936-8dd0-4544-af0a-c60d1ade9956 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:44:11.284466 n-d-853704-1 nova-conductor[120657]: ERROR nova.compute.rpcapi [None req-a505e936-8dd0-4544-af0a-c60d1ade9956 None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 18 23:44:11.284858 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-a505e936-8dd0-4544-af0a-c60d1ade9956 None None] Releasing lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 18 23:44:11.285746 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server [None req-a505e936-8dd0-4544-af0a-c60d1ade9956 None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 23:44:11.285746 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 23:44:11.285746 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 18 23:44:11.285746 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 18 23:44:11.285746 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 18 23:44:11.285746 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 23:44:11.285746 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 18 23:44:11.285746 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 23:44:11.285746 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 23:44:11.285746 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 18 23:44:11.285746 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 18 23:44:11.285746 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 18 23:44:11.285746 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 18 23:44:11.285746 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 18 23:44:11.285746 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 18 23:44:11.285746 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 18 23:44:11.285746 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 18 23:44:11.285746 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 18 23:44:11.285746 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 18 23:44:11.285746 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 18 23:44:11.285746 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 18 23:44:11.285746 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 18 23:44:11.285746 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 18 23:44:11.285746 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 23:44:11.285746 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 23:44:21.115526 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=13,nova_api:UPDATE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:44:21.193406 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=11,nova_api:DELETE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:44:21.282048 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:44:21.282801 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell1:SELECT=11,nova_cell1:SAVEPOINT=1,nova_cell1:INSERT=18,nova_cell1:RELEASE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:44:23.475696 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=14,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:44:43.444424 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=7,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:44:43.486078 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=3,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:44:50.772768 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-f852f15c-879e-4524-9d2d-08148ae1b5cd tempest-TestVolumeBootPattern-1034437477 tempest-TestVolumeBootPattern-1034437477-project] Took 0.44 seconds to select destinations for 1 instance(s). {{(pid=120658) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 18 23:44:50.793899 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-f852f15c-879e-4524-9d2d-08148ae1b5cd tempest-TestVolumeBootPattern-1034437477 tempest-TestVolumeBootPattern-1034437477-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:44:50.797400 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-f852f15c-879e-4524-9d2d-08148ae1b5cd tempest-TestVolumeBootPattern-1034437477 tempest-TestVolumeBootPattern-1034437477-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:44:50.797400 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-f852f15c-879e-4524-9d2d-08148ae1b5cd tempest-TestVolumeBootPattern-1034437477 tempest-TestVolumeBootPattern-1034437477-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:44:50.905168 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-f852f15c-879e-4524-9d2d-08148ae1b5cd tempest-TestVolumeBootPattern-1034437477 tempest-TestVolumeBootPattern-1034437477-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:44:50.905676 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-f852f15c-879e-4524-9d2d-08148ae1b5cd tempest-TestVolumeBootPattern-1034437477 tempest-TestVolumeBootPattern-1034437477-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:44:50.906194 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-f852f15c-879e-4524-9d2d-08148ae1b5cd tempest-TestVolumeBootPattern-1034437477 tempest-TestVolumeBootPattern-1034437477-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:44:50.906922 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-f852f15c-879e-4524-9d2d-08148ae1b5cd tempest-TestVolumeBootPattern-1034437477 tempest-TestVolumeBootPattern-1034437477-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:44:50.907449 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-f852f15c-879e-4524-9d2d-08148ae1b5cd tempest-TestVolumeBootPattern-1034437477 tempest-TestVolumeBootPattern-1034437477-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:44:50.907915 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-f852f15c-879e-4524-9d2d-08148ae1b5cd tempest-TestVolumeBootPattern-1034437477 tempest-TestVolumeBootPattern-1034437477-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:44:50.924195 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-f852f15c-879e-4524-9d2d-08148ae1b5cd tempest-TestVolumeBootPattern-1034437477 tempest-TestVolumeBootPattern-1034437477-project] Getting quotas for project dfab7231c21e4b1f82cb2fbf1c4e0986. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 18 23:44:50.929043 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-f852f15c-879e-4524-9d2d-08148ae1b5cd tempest-TestVolumeBootPattern-1034437477 tempest-TestVolumeBootPattern-1034437477-project] Getting quotas for user 480cd2375ac24f8a89b9730fcaa6965e and project dfab7231c21e4b1f82cb2fbf1c4e0986. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 18 23:44:50.939820 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-f852f15c-879e-4524-9d2d-08148ae1b5cd tempest-TestVolumeBootPattern-1034437477 tempest-TestVolumeBootPattern-1034437477-project] [instance: e4ba08f4-985b-4f89-9564-0cac9e8e1f54] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120658) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 18 23:44:50.940581 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-f852f15c-879e-4524-9d2d-08148ae1b5cd tempest-TestVolumeBootPattern-1034437477 tempest-TestVolumeBootPattern-1034437477-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:44:50.941009 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-f852f15c-879e-4524-9d2d-08148ae1b5cd tempest-TestVolumeBootPattern-1034437477 tempest-TestVolumeBootPattern-1034437477-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:44:50.941401 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-f852f15c-879e-4524-9d2d-08148ae1b5cd tempest-TestVolumeBootPattern-1034437477 tempest-TestVolumeBootPattern-1034437477-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:44:51.000211 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-f852f15c-879e-4524-9d2d-08148ae1b5cd tempest-TestVolumeBootPattern-1034437477 tempest-TestVolumeBootPattern-1034437477-project] [instance: e4ba08f4-985b-4f89-9564-0cac9e8e1f54] block_device_mapping [BlockDeviceMapping(attachment_id=cbe1b2a8-16f9-402b-b8b6-dd3f0425583b,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='volume',device_name=None,device_type=None,disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id=None,instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='volume',tag=None,updated_at=,uuid=,volume_id='1dab58f5-1506-4004-a96f-656f757b7450',volume_size=1,volume_type=None)] {{(pid=120658) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 18 23:44:51.001513 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-f852f15c-879e-4524-9d2d-08148ae1b5cd tempest-TestVolumeBootPattern-1034437477 tempest-TestVolumeBootPattern-1034437477-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:44:51.001959 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-f852f15c-879e-4524-9d2d-08148ae1b5cd tempest-TestVolumeBootPattern-1034437477 tempest-TestVolumeBootPattern-1034437477-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:44:51.002378 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-f852f15c-879e-4524-9d2d-08148ae1b5cd tempest-TestVolumeBootPattern-1034437477 tempest-TestVolumeBootPattern-1034437477-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:44:51.178364 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-f852f15c-879e-4524-9d2d-08148ae1b5cd tempest-TestVolumeBootPattern-1034437477 tempest-TestVolumeBootPattern-1034437477-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:44:51.178806 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-f852f15c-879e-4524-9d2d-08148ae1b5cd tempest-TestVolumeBootPattern-1034437477 tempest-TestVolumeBootPattern-1034437477-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:44:51.179239 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-f852f15c-879e-4524-9d2d-08148ae1b5cd tempest-TestVolumeBootPattern-1034437477 tempest-TestVolumeBootPattern-1034437477-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:44:51.179737 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-f852f15c-879e-4524-9d2d-08148ae1b5cd tempest-TestVolumeBootPattern-1034437477 tempest-TestVolumeBootPattern-1034437477-project] Acquiring lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 18 23:44:51.180076 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-f852f15c-879e-4524-9d2d-08148ae1b5cd tempest-TestVolumeBootPattern-1034437477 tempest-TestVolumeBootPattern-1034437477-project] Acquired lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 18 23:44:51.180812 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-4f7e6e2e-c5ef-42cd-951c-10c91591a2e8 None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:44:51.181360 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-4f7e6e2e-c5ef-42cd-951c-10c91591a2e8 None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:44:51.181746 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-4f7e6e2e-c5ef-42cd-951c-10c91591a2e8 None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:44:51.182519 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-4f7e6e2e-c5ef-42cd-951c-10c91591a2e8 None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:44:51.182929 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-4f7e6e2e-c5ef-42cd-951c-10c91591a2e8 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:44:51.183348 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-4f7e6e2e-c5ef-42cd-951c-10c91591a2e8 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:44:51.192214 n-d-853704-1 nova-conductor[120658]: ERROR nova.compute.rpcapi [None req-4f7e6e2e-c5ef-42cd-951c-10c91591a2e8 None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 18 23:44:51.192601 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-4f7e6e2e-c5ef-42cd-951c-10c91591a2e8 None None] Releasing lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 18 23:44:51.194228 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server [None req-4f7e6e2e-c5ef-42cd-951c-10c91591a2e8 None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 23:44:51.194228 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 23:44:51.194228 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 18 23:44:51.194228 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 18 23:44:51.194228 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 18 23:44:51.194228 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 23:44:51.194228 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 18 23:44:51.194228 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 23:44:51.194228 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 23:44:51.194228 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 18 23:44:51.194228 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 18 23:44:51.194228 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 18 23:44:51.194228 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 18 23:44:51.194228 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 18 23:44:51.194228 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 18 23:44:51.194228 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 18 23:44:51.194228 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 18 23:44:51.194228 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 18 23:44:51.194228 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 18 23:44:51.194228 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 18 23:44:51.194228 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 18 23:44:51.194228 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 18 23:44:51.194228 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 18 23:44:51.194228 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 23:44:51.194228 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 23:45:01.074066 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=11,nova_api:UPDATE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:45:01.128968 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=13,nova_api:DELETE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:45:01.188514 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:45:01.190211 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell1:SELECT=11,nova_cell1:INSERT=16 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:45:03.450214 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=10,nova_cell0:UPDATE=2 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:45:13.496070 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=6,nova_cell0:UPDATE=3 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:45:43.456275 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:45:53.498600 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:46:13.071396 n-d-853704-1 nova-conductor[120657]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 18 23:46:13.462819 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=5,nova_cell0:UPDATE=3 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:46:13.504230 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=3,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:46:33.468675 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=4,nova_cell0:UPDATE=2 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:46:53.478897 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=4,nova_cell0:UPDATE=2 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:46:53.518093 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=3 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:46:55.434126 n-d-853704-1 nova-conductor[120657]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 18 23:47:33.478388 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:47:33.514461 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:47:53.523820 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=3,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:48:13.474890 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:48:24.861149 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-58445faf-8f81-487e-9c65-2fdd1a34ee64 tempest-TestSecurityGroupsBasicOps-1788920570 tempest-TestSecurityGroupsBasicOps-1788920570-project] Took 0.54 seconds to select destinations for 1 instance(s). {{(pid=120657) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 18 23:48:24.883944 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-58445faf-8f81-487e-9c65-2fdd1a34ee64 tempest-TestSecurityGroupsBasicOps-1788920570 tempest-TestSecurityGroupsBasicOps-1788920570-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:48:24.884388 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-58445faf-8f81-487e-9c65-2fdd1a34ee64 tempest-TestSecurityGroupsBasicOps-1788920570 tempest-TestSecurityGroupsBasicOps-1788920570-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:48:24.884785 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-58445faf-8f81-487e-9c65-2fdd1a34ee64 tempest-TestSecurityGroupsBasicOps-1788920570 tempest-TestSecurityGroupsBasicOps-1788920570-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:48:25.041126 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:48:25.043222 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-58445faf-8f81-487e-9c65-2fdd1a34ee64 tempest-TestSecurityGroupsBasicOps-1788920570 tempest-TestSecurityGroupsBasicOps-1788920570-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:48:25.043655 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-58445faf-8f81-487e-9c65-2fdd1a34ee64 tempest-TestSecurityGroupsBasicOps-1788920570 tempest-TestSecurityGroupsBasicOps-1788920570-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:48:25.044061 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-58445faf-8f81-487e-9c65-2fdd1a34ee64 tempest-TestSecurityGroupsBasicOps-1788920570 tempest-TestSecurityGroupsBasicOps-1788920570-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:48:25.044735 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-58445faf-8f81-487e-9c65-2fdd1a34ee64 tempest-TestSecurityGroupsBasicOps-1788920570 tempest-TestSecurityGroupsBasicOps-1788920570-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:48:25.045143 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-58445faf-8f81-487e-9c65-2fdd1a34ee64 tempest-TestSecurityGroupsBasicOps-1788920570 tempest-TestSecurityGroupsBasicOps-1788920570-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:48:25.045529 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-58445faf-8f81-487e-9c65-2fdd1a34ee64 tempest-TestSecurityGroupsBasicOps-1788920570 tempest-TestSecurityGroupsBasicOps-1788920570-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:48:25.061730 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-58445faf-8f81-487e-9c65-2fdd1a34ee64 tempest-TestSecurityGroupsBasicOps-1788920570 tempest-TestSecurityGroupsBasicOps-1788920570-project] Getting quotas for project c04f872ea0d9419ea47391b5e6099f66. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 18 23:48:25.066259 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-58445faf-8f81-487e-9c65-2fdd1a34ee64 tempest-TestSecurityGroupsBasicOps-1788920570 tempest-TestSecurityGroupsBasicOps-1788920570-project] Getting quotas for user 135006f2ea914521941aef99a74a94f5 and project c04f872ea0d9419ea47391b5e6099f66. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 18 23:48:25.078165 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-58445faf-8f81-487e-9c65-2fdd1a34ee64 tempest-TestSecurityGroupsBasicOps-1788920570 tempest-TestSecurityGroupsBasicOps-1788920570-project] [instance: 4baf1fc8-f997-455c-929a-bb322daace6f] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120657) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 18 23:48:25.078973 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-58445faf-8f81-487e-9c65-2fdd1a34ee64 tempest-TestSecurityGroupsBasicOps-1788920570 tempest-TestSecurityGroupsBasicOps-1788920570-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:48:25.079400 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-58445faf-8f81-487e-9c65-2fdd1a34ee64 tempest-TestSecurityGroupsBasicOps-1788920570 tempest-TestSecurityGroupsBasicOps-1788920570-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:48:25.079829 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-58445faf-8f81-487e-9c65-2fdd1a34ee64 tempest-TestSecurityGroupsBasicOps-1788920570 tempest-TestSecurityGroupsBasicOps-1788920570-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:48:25.146591 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-58445faf-8f81-487e-9c65-2fdd1a34ee64 tempest-TestSecurityGroupsBasicOps-1788920570 tempest-TestSecurityGroupsBasicOps-1788920570-project] [instance: 4baf1fc8-f997-455c-929a-bb322daace6f] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='87b0fec9-a948-41f9-a3c0-f0a6a5a8f1e7',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120657) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 18 23:48:25.147860 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-58445faf-8f81-487e-9c65-2fdd1a34ee64 tempest-TestSecurityGroupsBasicOps-1788920570 tempest-TestSecurityGroupsBasicOps-1788920570-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:48:25.148297 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-58445faf-8f81-487e-9c65-2fdd1a34ee64 tempest-TestSecurityGroupsBasicOps-1788920570 tempest-TestSecurityGroupsBasicOps-1788920570-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:48:25.148694 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-58445faf-8f81-487e-9c65-2fdd1a34ee64 tempest-TestSecurityGroupsBasicOps-1788920570 tempest-TestSecurityGroupsBasicOps-1788920570-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:48:25.346925 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-58445faf-8f81-487e-9c65-2fdd1a34ee64 tempest-TestSecurityGroupsBasicOps-1788920570 tempest-TestSecurityGroupsBasicOps-1788920570-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:48:25.347380 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-58445faf-8f81-487e-9c65-2fdd1a34ee64 tempest-TestSecurityGroupsBasicOps-1788920570 tempest-TestSecurityGroupsBasicOps-1788920570-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:48:25.347881 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-58445faf-8f81-487e-9c65-2fdd1a34ee64 tempest-TestSecurityGroupsBasicOps-1788920570 tempest-TestSecurityGroupsBasicOps-1788920570-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:48:25.348435 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-58445faf-8f81-487e-9c65-2fdd1a34ee64 tempest-TestSecurityGroupsBasicOps-1788920570 tempest-TestSecurityGroupsBasicOps-1788920570-project] Acquiring lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 18 23:48:25.348795 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-58445faf-8f81-487e-9c65-2fdd1a34ee64 tempest-TestSecurityGroupsBasicOps-1788920570 tempest-TestSecurityGroupsBasicOps-1788920570-project] Acquired lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 18 23:48:25.349758 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-87fdc588-d7ca-4817-89d4-48d8a913d345 None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:48:25.350177 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-87fdc588-d7ca-4817-89d4-48d8a913d345 None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:48:25.350591 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-87fdc588-d7ca-4817-89d4-48d8a913d345 None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:48:25.351243 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-87fdc588-d7ca-4817-89d4-48d8a913d345 None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:48:25.351656 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-87fdc588-d7ca-4817-89d4-48d8a913d345 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:48:25.353080 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-87fdc588-d7ca-4817-89d4-48d8a913d345 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:48:25.360681 n-d-853704-1 nova-conductor[120657]: ERROR nova.compute.rpcapi [None req-87fdc588-d7ca-4817-89d4-48d8a913d345 None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 18 23:48:25.361071 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-87fdc588-d7ca-4817-89d4-48d8a913d345 None None] Releasing lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 18 23:48:25.362033 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server [None req-87fdc588-d7ca-4817-89d4-48d8a913d345 None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 23:48:25.362033 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 23:48:25.362033 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 18 23:48:25.362033 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 18 23:48:25.362033 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 18 23:48:25.362033 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 23:48:25.362033 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 18 23:48:25.362033 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 23:48:25.362033 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 23:48:25.362033 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 18 23:48:25.362033 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 18 23:48:25.362033 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 18 23:48:25.362033 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 18 23:48:25.362033 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 18 23:48:25.362033 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 18 23:48:25.362033 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 18 23:48:25.362033 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 18 23:48:25.362033 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 18 23:48:25.362033 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 18 23:48:25.362033 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 18 23:48:25.362033 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 18 23:48:25.362033 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 18 23:48:25.362033 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 18 23:48:25.362033 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 23:48:25.362033 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 23:48:35.302570 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=12,nova_api:UPDATE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:48:35.305453 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=12,nova_api:DELETE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:48:35.357494 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:48:35.359157 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell1:SELECT=11,nova_cell1:SAVEPOINT=1,nova_cell1:INSERT=18,nova_cell1:RELEASE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:48:53.468119 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:48:53.520825 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:49:13.474342 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=3,nova_cell0:UPDATE=2 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:49:33.515258 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:49:53.488812 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:49:54.159036 n-d-853704-1 nova-conductor[120658]: ERROR oslo.messaging._drivers.impl_rabbit [None req-20aed079-96ae-4758-85ad-fb7b9734fac7 tempest-TestNetworkAdvancedServerOps-1005354344 tempest-TestNetworkAdvancedServerOps-1005354344-project] [fe4e6e0b-b1ce-4fa1-b3d0-5231d7497da8] AMQP server on 10.222.0.49:5672 is unreachable: [Errno 104] Connection reset by peer. Trying again in 1 seconds.: ConnectionResetError: [Errno 104] Connection reset by peer Aug 18 23:49:55.164463 n-d-853704-1 nova-conductor[120658]: INFO oslo.messaging._drivers.impl_rabbit [None req-20aed079-96ae-4758-85ad-fb7b9734fac7 tempest-TestNetworkAdvancedServerOps-1005354344 tempest-TestNetworkAdvancedServerOps-1005354344-project] [fe4e6e0b-b1ce-4fa1-b3d0-5231d7497da8] Reconnected to AMQP server on 10.222.0.49:5672 via [amqp] client with port 42448. Aug 18 23:49:57.608197 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-20aed079-96ae-4758-85ad-fb7b9734fac7 tempest-TestNetworkAdvancedServerOps-1005354344 tempest-TestNetworkAdvancedServerOps-1005354344-project] Took 3.46 seconds to select destinations for 1 instance(s). {{(pid=120658) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 18 23:49:57.629548 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-20aed079-96ae-4758-85ad-fb7b9734fac7 tempest-TestNetworkAdvancedServerOps-1005354344 tempest-TestNetworkAdvancedServerOps-1005354344-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:49:57.630045 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-20aed079-96ae-4758-85ad-fb7b9734fac7 tempest-TestNetworkAdvancedServerOps-1005354344 tempest-TestNetworkAdvancedServerOps-1005354344-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:49:57.630377 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-20aed079-96ae-4758-85ad-fb7b9734fac7 tempest-TestNetworkAdvancedServerOps-1005354344 tempest-TestNetworkAdvancedServerOps-1005354344-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:49:57.758303 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-20aed079-96ae-4758-85ad-fb7b9734fac7 tempest-TestNetworkAdvancedServerOps-1005354344 tempest-TestNetworkAdvancedServerOps-1005354344-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:49:57.758752 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-20aed079-96ae-4758-85ad-fb7b9734fac7 tempest-TestNetworkAdvancedServerOps-1005354344 tempest-TestNetworkAdvancedServerOps-1005354344-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:49:57.759153 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-20aed079-96ae-4758-85ad-fb7b9734fac7 tempest-TestNetworkAdvancedServerOps-1005354344 tempest-TestNetworkAdvancedServerOps-1005354344-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:49:57.759873 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-20aed079-96ae-4758-85ad-fb7b9734fac7 tempest-TestNetworkAdvancedServerOps-1005354344 tempest-TestNetworkAdvancedServerOps-1005354344-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:49:57.760288 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-20aed079-96ae-4758-85ad-fb7b9734fac7 tempest-TestNetworkAdvancedServerOps-1005354344 tempest-TestNetworkAdvancedServerOps-1005354344-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:49:57.760675 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-20aed079-96ae-4758-85ad-fb7b9734fac7 tempest-TestNetworkAdvancedServerOps-1005354344 tempest-TestNetworkAdvancedServerOps-1005354344-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:49:57.778574 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-20aed079-96ae-4758-85ad-fb7b9734fac7 tempest-TestNetworkAdvancedServerOps-1005354344 tempest-TestNetworkAdvancedServerOps-1005354344-project] Getting quotas for project 76e04ea1eac74b8eaf7062d92184c480. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 18 23:49:57.783177 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-20aed079-96ae-4758-85ad-fb7b9734fac7 tempest-TestNetworkAdvancedServerOps-1005354344 tempest-TestNetworkAdvancedServerOps-1005354344-project] Getting quotas for user 46860dd540c9442e904c6404a08140c4 and project 76e04ea1eac74b8eaf7062d92184c480. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 18 23:49:57.794889 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-20aed079-96ae-4758-85ad-fb7b9734fac7 tempest-TestNetworkAdvancedServerOps-1005354344 tempest-TestNetworkAdvancedServerOps-1005354344-project] [instance: 8d7b3609-e10c-44da-b291-9b05ccad4868] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120658) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 18 23:49:57.795707 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-20aed079-96ae-4758-85ad-fb7b9734fac7 tempest-TestNetworkAdvancedServerOps-1005354344 tempest-TestNetworkAdvancedServerOps-1005354344-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:49:57.796151 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-20aed079-96ae-4758-85ad-fb7b9734fac7 tempest-TestNetworkAdvancedServerOps-1005354344 tempest-TestNetworkAdvancedServerOps-1005354344-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:49:57.796547 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-20aed079-96ae-4758-85ad-fb7b9734fac7 tempest-TestNetworkAdvancedServerOps-1005354344 tempest-TestNetworkAdvancedServerOps-1005354344-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:49:57.836964 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-20aed079-96ae-4758-85ad-fb7b9734fac7 tempest-TestNetworkAdvancedServerOps-1005354344 tempest-TestNetworkAdvancedServerOps-1005354344-project] [instance: 8d7b3609-e10c-44da-b291-9b05ccad4868] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='87b0fec9-a948-41f9-a3c0-f0a6a5a8f1e7',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120658) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 18 23:49:57.838199 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-20aed079-96ae-4758-85ad-fb7b9734fac7 tempest-TestNetworkAdvancedServerOps-1005354344 tempest-TestNetworkAdvancedServerOps-1005354344-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:49:57.838635 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-20aed079-96ae-4758-85ad-fb7b9734fac7 tempest-TestNetworkAdvancedServerOps-1005354344 tempest-TestNetworkAdvancedServerOps-1005354344-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:49:57.839052 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-20aed079-96ae-4758-85ad-fb7b9734fac7 tempest-TestNetworkAdvancedServerOps-1005354344 tempest-TestNetworkAdvancedServerOps-1005354344-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:49:57.995983 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-20aed079-96ae-4758-85ad-fb7b9734fac7 tempest-TestNetworkAdvancedServerOps-1005354344 tempest-TestNetworkAdvancedServerOps-1005354344-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:49:57.997058 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-20aed079-96ae-4758-85ad-fb7b9734fac7 tempest-TestNetworkAdvancedServerOps-1005354344 tempest-TestNetworkAdvancedServerOps-1005354344-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:49:57.997452 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-20aed079-96ae-4758-85ad-fb7b9734fac7 tempest-TestNetworkAdvancedServerOps-1005354344 tempest-TestNetworkAdvancedServerOps-1005354344-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:49:57.997983 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-20aed079-96ae-4758-85ad-fb7b9734fac7 tempest-TestNetworkAdvancedServerOps-1005354344 tempest-TestNetworkAdvancedServerOps-1005354344-project] Acquiring lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 18 23:49:57.998334 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-20aed079-96ae-4758-85ad-fb7b9734fac7 tempest-TestNetworkAdvancedServerOps-1005354344 tempest-TestNetworkAdvancedServerOps-1005354344-project] Acquired lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 18 23:49:57.999055 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-3613dd33-fa7c-4692-b17a-083b8474f4e0 None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:49:57.999487 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-3613dd33-fa7c-4692-b17a-083b8474f4e0 None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:49:57.999899 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-3613dd33-fa7c-4692-b17a-083b8474f4e0 None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:49:58.000540 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-3613dd33-fa7c-4692-b17a-083b8474f4e0 None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:49:58.000941 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-3613dd33-fa7c-4692-b17a-083b8474f4e0 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:49:58.001315 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-3613dd33-fa7c-4692-b17a-083b8474f4e0 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:49:58.009654 n-d-853704-1 nova-conductor[120658]: ERROR nova.compute.rpcapi [None req-3613dd33-fa7c-4692-b17a-083b8474f4e0 None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 18 23:49:58.010033 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-3613dd33-fa7c-4692-b17a-083b8474f4e0 None None] Releasing lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 18 23:49:58.010927 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server [None req-3613dd33-fa7c-4692-b17a-083b8474f4e0 None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 23:49:58.010927 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 23:49:58.010927 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 18 23:49:58.010927 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 18 23:49:58.010927 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 18 23:49:58.010927 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 23:49:58.010927 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 18 23:49:58.010927 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 23:49:58.010927 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 23:49:58.010927 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 18 23:49:58.010927 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 18 23:49:58.010927 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 18 23:49:58.010927 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 18 23:49:58.010927 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 18 23:49:58.010927 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 18 23:49:58.010927 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 18 23:49:58.010927 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 18 23:49:58.010927 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 18 23:49:58.010927 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 18 23:49:58.010927 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 18 23:49:58.010927 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 18 23:49:58.010927 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 18 23:49:58.010927 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 18 23:49:58.010927 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 23:49:58.010927 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 18 23:50:07.936839 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=10 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:50:07.937486 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=14,nova_api:UPDATE=1,nova_api:DELETE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:50:08.006634 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:50:08.007808 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell1:SELECT=11,nova_cell1:INSERT=17 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:50:13.523305 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:50:15.443597 n-d-853704-1 nova-conductor[120657]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 18 23:50:23.493595 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=12,nova_cell0:UPDATE=3 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:50:33.524945 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=3,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:50:43.498240 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=4,nova_cell0:UPDATE=2 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:50:53.527123 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=4,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:51:13.081682 n-d-853704-1 nova-conductor[120657]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: Server unexpectedly closed connection Aug 18 23:51:23.500616 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:51:33.524464 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:52:03.502523 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:52:13.530179 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=7,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:52:43.488826 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:52:53.531529 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:53:13.531088 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=3,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:53:23.492488 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:53:33.534724 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=4,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:53:35.453057 n-d-853704-1 nova-conductor[120657]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 18 23:53:43.502000 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=3,nova_cell0:UPDATE=2 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:54:13.531134 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:54:23.500963 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:54:33.089086 n-d-853704-1 nova-conductor[120657]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 18 23:54:33.536304 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=3,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:55:01.372598 n-d-853704-1 nova-conductor[120658]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 18 23:55:03.505644 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:55:13.538599 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:55:33.511577 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=5,nova_cell0:UPDATE=3 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:55:53.542869 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:56:03.515397 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=6,nova_cell0:UPDATE=3 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:56:13.549721 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=3,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:56:33.526342 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=6,nova_cell0:UPDATE=3 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:56:43.562470 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=6,nova_cell0:UPDATE=3 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:56:55.463287 n-d-853704-1 nova-conductor[120657]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 18 23:57:03.571410 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=4,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:57:13.528677 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:57:16.869297 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-e7898ea0-8e3d-425a-ae67-baa81ec269ea tempest-TestVolumeBootPattern-1034437477 tempest-TestVolumeBootPattern-1034437477-project] Took 0.43 seconds to select destinations for 1 instance(s). {{(pid=120657) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 18 23:57:16.890248 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-e7898ea0-8e3d-425a-ae67-baa81ec269ea tempest-TestVolumeBootPattern-1034437477 tempest-TestVolumeBootPattern-1034437477-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:57:16.890729 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-e7898ea0-8e3d-425a-ae67-baa81ec269ea tempest-TestVolumeBootPattern-1034437477 tempest-TestVolumeBootPattern-1034437477-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:57:16.891121 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-e7898ea0-8e3d-425a-ae67-baa81ec269ea tempest-TestVolumeBootPattern-1034437477 tempest-TestVolumeBootPattern-1034437477-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:57:17.017405 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-e7898ea0-8e3d-425a-ae67-baa81ec269ea tempest-TestVolumeBootPattern-1034437477 tempest-TestVolumeBootPattern-1034437477-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:57:17.017847 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-e7898ea0-8e3d-425a-ae67-baa81ec269ea tempest-TestVolumeBootPattern-1034437477 tempest-TestVolumeBootPattern-1034437477-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:57:17.018244 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-e7898ea0-8e3d-425a-ae67-baa81ec269ea tempest-TestVolumeBootPattern-1034437477 tempest-TestVolumeBootPattern-1034437477-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:57:17.018907 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-e7898ea0-8e3d-425a-ae67-baa81ec269ea tempest-TestVolumeBootPattern-1034437477 tempest-TestVolumeBootPattern-1034437477-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:57:17.019323 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-e7898ea0-8e3d-425a-ae67-baa81ec269ea tempest-TestVolumeBootPattern-1034437477 tempest-TestVolumeBootPattern-1034437477-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:57:17.019729 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-e7898ea0-8e3d-425a-ae67-baa81ec269ea tempest-TestVolumeBootPattern-1034437477 tempest-TestVolumeBootPattern-1034437477-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:57:17.037708 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-e7898ea0-8e3d-425a-ae67-baa81ec269ea tempest-TestVolumeBootPattern-1034437477 tempest-TestVolumeBootPattern-1034437477-project] Getting quotas for project dfab7231c21e4b1f82cb2fbf1c4e0986. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 18 23:57:17.042290 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-e7898ea0-8e3d-425a-ae67-baa81ec269ea tempest-TestVolumeBootPattern-1034437477 tempest-TestVolumeBootPattern-1034437477-project] Getting quotas for user 480cd2375ac24f8a89b9730fcaa6965e and project dfab7231c21e4b1f82cb2fbf1c4e0986. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 18 23:57:17.053686 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-e7898ea0-8e3d-425a-ae67-baa81ec269ea tempest-TestVolumeBootPattern-1034437477 tempest-TestVolumeBootPattern-1034437477-project] [instance: dcccc046-5ab3-4d16-8882-99d6aac13b5e] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120657) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 18 23:57:17.054438 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-e7898ea0-8e3d-425a-ae67-baa81ec269ea tempest-TestVolumeBootPattern-1034437477 tempest-TestVolumeBootPattern-1034437477-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:57:17.054869 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-e7898ea0-8e3d-425a-ae67-baa81ec269ea tempest-TestVolumeBootPattern-1034437477 tempest-TestVolumeBootPattern-1034437477-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:57:17.055262 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-e7898ea0-8e3d-425a-ae67-baa81ec269ea tempest-TestVolumeBootPattern-1034437477 tempest-TestVolumeBootPattern-1034437477-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:57:17.110048 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-e7898ea0-8e3d-425a-ae67-baa81ec269ea tempest-TestVolumeBootPattern-1034437477 tempest-TestVolumeBootPattern-1034437477-project] [instance: dcccc046-5ab3-4d16-8882-99d6aac13b5e] block_device_mapping [BlockDeviceMapping(attachment_id=ab79cf8d-68a5-4eb7-a01c-96918a5c31d6,boot_index=0,connection_info=None,created_at=,delete_on_termination=False,deleted=,deleted_at=,destination_type='volume',device_name=None,device_type=None,disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id=None,instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='volume',tag=None,updated_at=,uuid=,volume_id='2ef01d53-f000-4361-afde-3c93d51e8406',volume_size=1,volume_type=None)] {{(pid=120657) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 18 23:57:17.111327 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-e7898ea0-8e3d-425a-ae67-baa81ec269ea tempest-TestVolumeBootPattern-1034437477 tempest-TestVolumeBootPattern-1034437477-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:57:17.111807 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-e7898ea0-8e3d-425a-ae67-baa81ec269ea tempest-TestVolumeBootPattern-1034437477 tempest-TestVolumeBootPattern-1034437477-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:57:17.112230 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-e7898ea0-8e3d-425a-ae67-baa81ec269ea tempest-TestVolumeBootPattern-1034437477 tempest-TestVolumeBootPattern-1034437477-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:57:17.373001 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-e7898ea0-8e3d-425a-ae67-baa81ec269ea tempest-TestVolumeBootPattern-1034437477 tempest-TestVolumeBootPattern-1034437477-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:57:17.373357 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-e7898ea0-8e3d-425a-ae67-baa81ec269ea tempest-TestVolumeBootPattern-1034437477 tempest-TestVolumeBootPattern-1034437477-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:57:17.373780 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-e7898ea0-8e3d-425a-ae67-baa81ec269ea tempest-TestVolumeBootPattern-1034437477 tempest-TestVolumeBootPattern-1034437477-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:57:17.374286 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-e7898ea0-8e3d-425a-ae67-baa81ec269ea tempest-TestVolumeBootPattern-1034437477 tempest-TestVolumeBootPattern-1034437477-project] Acquiring lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 18 23:57:17.374640 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-e7898ea0-8e3d-425a-ae67-baa81ec269ea tempest-TestVolumeBootPattern-1034437477 tempest-TestVolumeBootPattern-1034437477-project] Acquired lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 18 23:57:17.375371 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-92127867-8a4d-4846-8c9a-1b71badad4b1 None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:57:17.375862 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-92127867-8a4d-4846-8c9a-1b71badad4b1 None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:57:17.376285 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-92127867-8a4d-4846-8c9a-1b71badad4b1 None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:57:17.376995 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-92127867-8a4d-4846-8c9a-1b71badad4b1 None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 18 23:57:17.377405 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-92127867-8a4d-4846-8c9a-1b71badad4b1 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 18 23:57:17.377785 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-92127867-8a4d-4846-8c9a-1b71badad4b1 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 18 23:57:17.385808 n-d-853704-1 nova-conductor[120657]: ERROR nova.compute.rpcapi [None req-92127867-8a4d-4846-8c9a-1b71badad4b1 None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 18 23:57:17.386193 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-92127867-8a4d-4846-8c9a-1b71badad4b1 None None] Releasing lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 18 23:57:17.387100 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server [None req-92127867-8a4d-4846-8c9a-1b71badad4b1 None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 23:57:17.387100 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 23:57:17.387100 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 18 23:57:17.387100 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 18 23:57:17.387100 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 18 23:57:17.387100 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 23:57:17.387100 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 18 23:57:17.387100 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 23:57:17.387100 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 18 23:57:17.387100 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 18 23:57:17.387100 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 18 23:57:17.387100 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 18 23:57:17.387100 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 18 23:57:17.387100 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 18 23:57:17.387100 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 18 23:57:17.387100 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 18 23:57:17.387100 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 18 23:57:17.387100 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 18 23:57:17.387100 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 18 23:57:17.387100 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 18 23:57:17.387100 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 18 23:57:17.387100 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 18 23:57:17.387100 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 18 23:57:17.387100 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 18 23:57:17.387100 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 18 23:57:27.317337 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=13,nova_api:UPDATE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:57:27.318153 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=11,nova_api:DELETE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:57:27.382742 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:57:27.383513 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell1:SELECT=11,nova_cell1:INSERT=16 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:57:43.576185 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=14,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:57:53.537393 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:58:03.583052 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=4,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:58:21.380977 n-d-853704-1 nova-conductor[120658]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 18 23:58:23.585775 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=4,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:58:33.543387 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:58:43.591158 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=4,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:59:13.548406 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:59:23.594545 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 18 23:59:33.098339 n-d-853704-1 nova-conductor[120657]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 18 23:59:43.557623 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=5,nova_cell0:UPDATE=3 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:00:03.595339 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=3 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:00:15.473165 n-d-853704-1 nova-conductor[120657]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 19 00:00:23.569825 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:00:33.595622 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=5,nova_cell0:UPDATE=3 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:01:03.575149 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:01:13.593794 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:01:41.388994 n-d-853704-1 nova-conductor[120658]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 19 00:01:43.574950 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:01:43.601889 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=5,nova_cell0:UPDATE=3 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:02:03.575452 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=3,nova_cell0:UPDATE=2 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:02:03.607478 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=4,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:02:43.576458 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:02:43.609026 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:02:53.108194 n-d-853704-1 nova-conductor[120657]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 19 00:03:13.584557 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=5,nova_cell0:UPDATE=3 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:03:13.616201 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=5,nova_cell0:UPDATE=3 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:03:35.481220 n-d-853704-1 nova-conductor[120657]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 19 00:03:43.583558 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=6,nova_cell0:UPDATE=3 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:03:53.617042 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:04:23.589847 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:04:33.625116 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=7,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:04:43.599198 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=3,nova_cell0:UPDATE=2 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:05:01.398287 n-d-853704-1 nova-conductor[120658]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 19 00:05:13.635085 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:05:23.605145 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:05:33.641073 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=3,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:06:03.613773 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=7,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:06:13.116163 n-d-853704-1 nova-conductor[120657]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 19 00:06:13.642203 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:06:33.649697 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=3,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:06:43.606951 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:06:53.650560 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=4,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:06:55.491763 n-d-853704-1 nova-conductor[120657]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 19 00:07:23.614050 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:07:33.651635 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:08:03.619528 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=7,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:08:03.660007 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=5,nova_cell0:UPDATE=3 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:08:21.407386 n-d-853704-1 nova-conductor[120658]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 19 00:08:43.630373 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:08:43.672652 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:09:03.633233 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=3,nova_cell0:UPDATE=2 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:09:03.673883 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=3,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:09:23.644294 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=4,nova_cell0:UPDATE=2 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:09:23.684550 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=4,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:09:33.125572 n-d-853704-1 nova-conductor[120657]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 19 00:09:41.784428 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-a790a15d-6834-47aa-8f6e-9e1e3f22e1fd tempest-ServersAdminNegativeTestJSON-1127241474 tempest-ServersAdminNegativeTestJSON-1127241474-project] Took 0.62 seconds to select destinations for 1 instance(s). {{(pid=120658) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 19 00:09:41.804056 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-a790a15d-6834-47aa-8f6e-9e1e3f22e1fd tempest-ServersAdminNegativeTestJSON-1127241474 tempest-ServersAdminNegativeTestJSON-1127241474-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 00:09:41.804741 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-a790a15d-6834-47aa-8f6e-9e1e3f22e1fd tempest-ServersAdminNegativeTestJSON-1127241474 tempest-ServersAdminNegativeTestJSON-1127241474-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 00:09:41.805149 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-a790a15d-6834-47aa-8f6e-9e1e3f22e1fd tempest-ServersAdminNegativeTestJSON-1127241474 tempest-ServersAdminNegativeTestJSON-1127241474-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 00:09:42.012315 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-a790a15d-6834-47aa-8f6e-9e1e3f22e1fd tempest-ServersAdminNegativeTestJSON-1127241474 tempest-ServersAdminNegativeTestJSON-1127241474-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 00:09:42.012668 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-a790a15d-6834-47aa-8f6e-9e1e3f22e1fd tempest-ServersAdminNegativeTestJSON-1127241474 tempest-ServersAdminNegativeTestJSON-1127241474-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 00:09:42.013056 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-a790a15d-6834-47aa-8f6e-9e1e3f22e1fd tempest-ServersAdminNegativeTestJSON-1127241474 tempest-ServersAdminNegativeTestJSON-1127241474-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 00:09:42.013747 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-a790a15d-6834-47aa-8f6e-9e1e3f22e1fd tempest-ServersAdminNegativeTestJSON-1127241474 tempest-ServersAdminNegativeTestJSON-1127241474-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 00:09:42.014891 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-a790a15d-6834-47aa-8f6e-9e1e3f22e1fd tempest-ServersAdminNegativeTestJSON-1127241474 tempest-ServersAdminNegativeTestJSON-1127241474-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 00:09:42.015271 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-a790a15d-6834-47aa-8f6e-9e1e3f22e1fd tempest-ServersAdminNegativeTestJSON-1127241474 tempest-ServersAdminNegativeTestJSON-1127241474-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 00:09:42.030401 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-a790a15d-6834-47aa-8f6e-9e1e3f22e1fd tempest-ServersAdminNegativeTestJSON-1127241474 tempest-ServersAdminNegativeTestJSON-1127241474-project] Getting quotas for project 6683a2b5b769467781409cc35df7c010. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 19 00:09:42.035376 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-a790a15d-6834-47aa-8f6e-9e1e3f22e1fd tempest-ServersAdminNegativeTestJSON-1127241474 tempest-ServersAdminNegativeTestJSON-1127241474-project] Getting quotas for user 9a3fdd35f482455cb162c9db005b242a and project 6683a2b5b769467781409cc35df7c010. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 19 00:09:42.046121 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-a790a15d-6834-47aa-8f6e-9e1e3f22e1fd tempest-ServersAdminNegativeTestJSON-1127241474 tempest-ServersAdminNegativeTestJSON-1127241474-project] [instance: aaf33eff-07e8-4a51-abcd-3a455690753d] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120658) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 19 00:09:42.046858 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-a790a15d-6834-47aa-8f6e-9e1e3f22e1fd tempest-ServersAdminNegativeTestJSON-1127241474 tempest-ServersAdminNegativeTestJSON-1127241474-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 00:09:42.047278 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-a790a15d-6834-47aa-8f6e-9e1e3f22e1fd tempest-ServersAdminNegativeTestJSON-1127241474 tempest-ServersAdminNegativeTestJSON-1127241474-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 00:09:42.047685 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-a790a15d-6834-47aa-8f6e-9e1e3f22e1fd tempest-ServersAdminNegativeTestJSON-1127241474 tempest-ServersAdminNegativeTestJSON-1127241474-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 00:09:42.098492 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-a790a15d-6834-47aa-8f6e-9e1e3f22e1fd tempest-ServersAdminNegativeTestJSON-1127241474 tempest-ServersAdminNegativeTestJSON-1127241474-project] [instance: aaf33eff-07e8-4a51-abcd-3a455690753d] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='87b0fec9-a948-41f9-a3c0-f0a6a5a8f1e7',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120658) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 19 00:09:42.099703 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-a790a15d-6834-47aa-8f6e-9e1e3f22e1fd tempest-ServersAdminNegativeTestJSON-1127241474 tempest-ServersAdminNegativeTestJSON-1127241474-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 00:09:42.100131 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-a790a15d-6834-47aa-8f6e-9e1e3f22e1fd tempest-ServersAdminNegativeTestJSON-1127241474 tempest-ServersAdminNegativeTestJSON-1127241474-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 00:09:42.100519 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-a790a15d-6834-47aa-8f6e-9e1e3f22e1fd tempest-ServersAdminNegativeTestJSON-1127241474 tempest-ServersAdminNegativeTestJSON-1127241474-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 00:09:42.297166 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-a790a15d-6834-47aa-8f6e-9e1e3f22e1fd tempest-ServersAdminNegativeTestJSON-1127241474 tempest-ServersAdminNegativeTestJSON-1127241474-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 00:09:42.297714 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-a790a15d-6834-47aa-8f6e-9e1e3f22e1fd tempest-ServersAdminNegativeTestJSON-1127241474 tempest-ServersAdminNegativeTestJSON-1127241474-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 00:09:42.298195 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-a790a15d-6834-47aa-8f6e-9e1e3f22e1fd tempest-ServersAdminNegativeTestJSON-1127241474 tempest-ServersAdminNegativeTestJSON-1127241474-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 00:09:42.298706 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-a790a15d-6834-47aa-8f6e-9e1e3f22e1fd tempest-ServersAdminNegativeTestJSON-1127241474 tempest-ServersAdminNegativeTestJSON-1127241474-project] Acquiring lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 19 00:09:42.299057 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-a790a15d-6834-47aa-8f6e-9e1e3f22e1fd tempest-ServersAdminNegativeTestJSON-1127241474 tempest-ServersAdminNegativeTestJSON-1127241474-project] Acquired lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 19 00:09:42.299892 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-56fb62bc-f72f-4c5a-b79e-48db406a9b8a None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 00:09:42.300296 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-56fb62bc-f72f-4c5a-b79e-48db406a9b8a None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 00:09:42.300674 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-56fb62bc-f72f-4c5a-b79e-48db406a9b8a None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 00:09:42.301314 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-56fb62bc-f72f-4c5a-b79e-48db406a9b8a None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 00:09:42.301714 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-56fb62bc-f72f-4c5a-b79e-48db406a9b8a None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 00:09:42.302087 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-56fb62bc-f72f-4c5a-b79e-48db406a9b8a None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 00:09:42.313113 n-d-853704-1 nova-conductor[120658]: ERROR nova.compute.rpcapi [None req-56fb62bc-f72f-4c5a-b79e-48db406a9b8a None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 19 00:09:42.313499 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-56fb62bc-f72f-4c5a-b79e-48db406a9b8a None None] Releasing lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 19 00:09:42.314422 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server [None req-56fb62bc-f72f-4c5a-b79e-48db406a9b8a None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 19 00:09:42.314422 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 19 00:09:42.314422 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 19 00:09:42.314422 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 19 00:09:42.314422 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 19 00:09:42.314422 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 19 00:09:42.314422 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 19 00:09:42.314422 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 19 00:09:42.314422 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 19 00:09:42.314422 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 19 00:09:42.314422 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 19 00:09:42.314422 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 19 00:09:42.314422 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 19 00:09:42.314422 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 19 00:09:42.314422 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 19 00:09:42.314422 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 19 00:09:42.314422 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 19 00:09:42.314422 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 19 00:09:42.314422 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 19 00:09:42.314422 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 19 00:09:42.314422 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 19 00:09:42.314422 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 19 00:09:42.314422 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 19 00:09:42.314422 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 19 00:09:42.314422 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 19 00:09:52.226524 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=11,nova_api:UPDATE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:09:52.227362 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=13,nova_api:DELETE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:09:52.311256 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:09:52.311890 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell1:SELECT=11,nova_cell1:SAVEPOINT=1,nova_cell1:INSERT=18,nova_cell1:RELEASE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:10:03.640709 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=15,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:10:03.681977 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:10:15.500790 n-d-853704-1 nova-conductor[120657]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 19 00:10:43.633101 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:10:43.677012 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:11:03.642651 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=3,nova_cell0:UPDATE=2 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:11:23.675149 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:11:43.639492 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:12:03.680726 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:12:23.641549 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:12:33.688844 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=5,nova_cell0:UPDATE=3 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:12:53.133855 n-d-853704-1 nova-conductor[120657]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 19 00:13:03.651587 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=7,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:13:13.694755 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:13:33.654571 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=6,nova_cell0:UPDATE=3 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:13:33.696208 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=3,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:13:35.509180 n-d-853704-1 nova-conductor[120657]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 19 00:14:03.706733 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=6,nova_cell0:UPDATE=3 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:14:13.665340 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:14:33.674201 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=3,nova_cell0:UPDATE=2 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:14:33.714228 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=6,nova_cell0:UPDATE=3 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:15:01.418428 n-d-853704-1 nova-conductor[120658]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 19 00:15:13.678588 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:15:13.719451 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:15:33.688742 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=3,nova_cell0:UPDATE=2 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:15:33.729542 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=3,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:15:53.388717 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-0e3fd4e1-e0c7-405a-94a8-19e91b08c009 tempest-DeleteServersTestJSON-1607516883 tempest-DeleteServersTestJSON-1607516883-project] Took 0.68 seconds to select destinations for 1 instance(s). {{(pid=120657) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 19 00:15:53.410378 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-0e3fd4e1-e0c7-405a-94a8-19e91b08c009 tempest-DeleteServersTestJSON-1607516883 tempest-DeleteServersTestJSON-1607516883-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 00:15:53.410842 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-0e3fd4e1-e0c7-405a-94a8-19e91b08c009 tempest-DeleteServersTestJSON-1607516883 tempest-DeleteServersTestJSON-1607516883-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 00:15:53.411209 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-0e3fd4e1-e0c7-405a-94a8-19e91b08c009 tempest-DeleteServersTestJSON-1607516883 tempest-DeleteServersTestJSON-1607516883-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 00:15:53.556852 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-0e3fd4e1-e0c7-405a-94a8-19e91b08c009 tempest-DeleteServersTestJSON-1607516883 tempest-DeleteServersTestJSON-1607516883-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 00:15:53.557205 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-0e3fd4e1-e0c7-405a-94a8-19e91b08c009 tempest-DeleteServersTestJSON-1607516883 tempest-DeleteServersTestJSON-1607516883-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 00:15:53.557591 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-0e3fd4e1-e0c7-405a-94a8-19e91b08c009 tempest-DeleteServersTestJSON-1607516883 tempest-DeleteServersTestJSON-1607516883-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 00:15:53.558253 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-0e3fd4e1-e0c7-405a-94a8-19e91b08c009 tempest-DeleteServersTestJSON-1607516883 tempest-DeleteServersTestJSON-1607516883-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 00:15:53.558658 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-0e3fd4e1-e0c7-405a-94a8-19e91b08c009 tempest-DeleteServersTestJSON-1607516883 tempest-DeleteServersTestJSON-1607516883-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 00:15:53.559036 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-0e3fd4e1-e0c7-405a-94a8-19e91b08c009 tempest-DeleteServersTestJSON-1607516883 tempest-DeleteServersTestJSON-1607516883-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 00:15:53.575435 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-0e3fd4e1-e0c7-405a-94a8-19e91b08c009 tempest-DeleteServersTestJSON-1607516883 tempest-DeleteServersTestJSON-1607516883-project] Getting quotas for project 18f91b43faef48728752b041bf19d0c0. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 19 00:15:53.580109 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-0e3fd4e1-e0c7-405a-94a8-19e91b08c009 tempest-DeleteServersTestJSON-1607516883 tempest-DeleteServersTestJSON-1607516883-project] Getting quotas for user 1d95996c78d04a58b91d3c8f42a810f5 and project 18f91b43faef48728752b041bf19d0c0. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 19 00:15:53.592069 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-0e3fd4e1-e0c7-405a-94a8-19e91b08c009 tempest-DeleteServersTestJSON-1607516883 tempest-DeleteServersTestJSON-1607516883-project] [instance: 00379bc2-cb84-49e9-9150-8f2df92cff20] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120657) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 19 00:15:53.592935 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-0e3fd4e1-e0c7-405a-94a8-19e91b08c009 tempest-DeleteServersTestJSON-1607516883 tempest-DeleteServersTestJSON-1607516883-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 00:15:53.593373 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-0e3fd4e1-e0c7-405a-94a8-19e91b08c009 tempest-DeleteServersTestJSON-1607516883 tempest-DeleteServersTestJSON-1607516883-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 00:15:53.593767 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-0e3fd4e1-e0c7-405a-94a8-19e91b08c009 tempest-DeleteServersTestJSON-1607516883 tempest-DeleteServersTestJSON-1607516883-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 00:15:53.688344 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-0e3fd4e1-e0c7-405a-94a8-19e91b08c009 tempest-DeleteServersTestJSON-1607516883 tempest-DeleteServersTestJSON-1607516883-project] [instance: 00379bc2-cb84-49e9-9150-8f2df92cff20] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='87b0fec9-a948-41f9-a3c0-f0a6a5a8f1e7',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120657) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 19 00:15:53.689583 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-0e3fd4e1-e0c7-405a-94a8-19e91b08c009 tempest-DeleteServersTestJSON-1607516883 tempest-DeleteServersTestJSON-1607516883-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 00:15:53.690030 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-0e3fd4e1-e0c7-405a-94a8-19e91b08c009 tempest-DeleteServersTestJSON-1607516883 tempest-DeleteServersTestJSON-1607516883-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 00:15:53.690424 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-0e3fd4e1-e0c7-405a-94a8-19e91b08c009 tempest-DeleteServersTestJSON-1607516883 tempest-DeleteServersTestJSON-1607516883-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 00:15:53.958985 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-0e3fd4e1-e0c7-405a-94a8-19e91b08c009 tempest-DeleteServersTestJSON-1607516883 tempest-DeleteServersTestJSON-1607516883-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 00:15:53.959507 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-0e3fd4e1-e0c7-405a-94a8-19e91b08c009 tempest-DeleteServersTestJSON-1607516883 tempest-DeleteServersTestJSON-1607516883-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 00:15:53.959928 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-0e3fd4e1-e0c7-405a-94a8-19e91b08c009 tempest-DeleteServersTestJSON-1607516883 tempest-DeleteServersTestJSON-1607516883-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 00:15:53.960372 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-0e3fd4e1-e0c7-405a-94a8-19e91b08c009 tempest-DeleteServersTestJSON-1607516883 tempest-DeleteServersTestJSON-1607516883-project] Acquiring lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 19 00:15:53.960717 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-0e3fd4e1-e0c7-405a-94a8-19e91b08c009 tempest-DeleteServersTestJSON-1607516883 tempest-DeleteServersTestJSON-1607516883-project] Acquired lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 19 00:15:53.961441 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-9c8d6526-2ec5-4d01-821f-8d09a080b16b None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 00:15:53.962081 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-9c8d6526-2ec5-4d01-821f-8d09a080b16b None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 00:15:53.962402 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-9c8d6526-2ec5-4d01-821f-8d09a080b16b None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 00:15:53.963152 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-9c8d6526-2ec5-4d01-821f-8d09a080b16b None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 00:15:53.963563 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-9c8d6526-2ec5-4d01-821f-8d09a080b16b None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 00:15:53.963974 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-9c8d6526-2ec5-4d01-821f-8d09a080b16b None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 00:15:53.974097 n-d-853704-1 nova-conductor[120657]: ERROR nova.compute.rpcapi [None req-9c8d6526-2ec5-4d01-821f-8d09a080b16b None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 19 00:15:53.976959 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-9c8d6526-2ec5-4d01-821f-8d09a080b16b None None] Releasing lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 19 00:15:53.976959 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server [None req-9c8d6526-2ec5-4d01-821f-8d09a080b16b None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 19 00:15:53.976959 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 19 00:15:53.976959 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 19 00:15:53.976959 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 19 00:15:53.976959 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 19 00:15:53.976959 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 19 00:15:53.976959 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 19 00:15:53.976959 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 19 00:15:53.976959 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 19 00:15:53.976959 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 19 00:15:53.976959 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 19 00:15:53.976959 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 19 00:15:53.976959 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 19 00:15:53.976959 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 19 00:15:53.976959 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 19 00:15:53.976959 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 19 00:15:53.976959 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 19 00:15:53.976959 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 19 00:15:53.976959 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 19 00:15:53.976959 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 19 00:15:53.976959 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 19 00:15:53.976959 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 19 00:15:53.976959 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 19 00:15:53.976959 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 19 00:15:53.976959 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 19 00:16:03.822285 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=14,nova_api:UPDATE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:16:03.891926 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=10,nova_api:DELETE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:16:03.971251 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:16:03.972021 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell1:SELECT=11,nova_cell1:SAVEPOINT=1,nova_cell1:INSERT=18,nova_cell1:RELEASE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:16:13.698029 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:16:13.737801 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=15,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:16:53.694563 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:16:53.737104 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:16:55.517455 n-d-853704-1 nova-conductor[120657]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 19 00:17:13.738884 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=3,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:17:23.703417 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=5,nova_cell0:UPDATE=3 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:17:33.738909 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=4,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:17:43.713956 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=4,nova_cell0:UPDATE=2 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:18:13.735454 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:18:21.427720 n-d-853704-1 nova-conductor[120658]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 19 00:18:23.713822 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:18:53.738008 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:19:03.716939 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:19:33.722646 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=5,nova_cell0:UPDATE=3 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:19:33.743132 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=7,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:19:53.726292 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=4,nova_cell0:UPDATE=2 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:19:53.746676 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=4,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:20:13.727769 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=4,nova_cell0:UPDATE=2 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:20:13.748493 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=4,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:20:15.525830 n-d-853704-1 nova-conductor[120657]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 19 00:20:53.729266 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:20:53.751148 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:21:13.146305 n-d-853704-1 nova-conductor[120657]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 19 00:21:23.733386 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=5,nova_cell0:UPDATE=3 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:21:23.754576 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=5,nova_cell0:UPDATE=3 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:21:41.435094 n-d-853704-1 nova-conductor[120658]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 19 00:21:53.759693 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=6,nova_cell0:UPDATE=3 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:22:03.744929 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:22:05.349050 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-2ebdd804-c583-40b1-bd89-3bdaf0db037a tempest-ServerDiskConfigTestJSON-1882845406 tempest-ServerDiskConfigTestJSON-1882845406-project] Took 0.67 seconds to select destinations for 1 instance(s). {{(pid=120658) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 19 00:22:05.369039 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-2ebdd804-c583-40b1-bd89-3bdaf0db037a tempest-ServerDiskConfigTestJSON-1882845406 tempest-ServerDiskConfigTestJSON-1882845406-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 00:22:05.371949 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-2ebdd804-c583-40b1-bd89-3bdaf0db037a tempest-ServerDiskConfigTestJSON-1882845406 tempest-ServerDiskConfigTestJSON-1882845406-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 00:22:05.371949 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-2ebdd804-c583-40b1-bd89-3bdaf0db037a tempest-ServerDiskConfigTestJSON-1882845406 tempest-ServerDiskConfigTestJSON-1882845406-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 00:22:05.565926 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-2ebdd804-c583-40b1-bd89-3bdaf0db037a tempest-ServerDiskConfigTestJSON-1882845406 tempest-ServerDiskConfigTestJSON-1882845406-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 00:22:05.566396 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-2ebdd804-c583-40b1-bd89-3bdaf0db037a tempest-ServerDiskConfigTestJSON-1882845406 tempest-ServerDiskConfigTestJSON-1882845406-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 00:22:05.566847 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-2ebdd804-c583-40b1-bd89-3bdaf0db037a tempest-ServerDiskConfigTestJSON-1882845406 tempest-ServerDiskConfigTestJSON-1882845406-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 00:22:05.567469 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-2ebdd804-c583-40b1-bd89-3bdaf0db037a tempest-ServerDiskConfigTestJSON-1882845406 tempest-ServerDiskConfigTestJSON-1882845406-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 00:22:05.567964 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-2ebdd804-c583-40b1-bd89-3bdaf0db037a tempest-ServerDiskConfigTestJSON-1882845406 tempest-ServerDiskConfigTestJSON-1882845406-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 00:22:05.568345 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-2ebdd804-c583-40b1-bd89-3bdaf0db037a tempest-ServerDiskConfigTestJSON-1882845406 tempest-ServerDiskConfigTestJSON-1882845406-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 00:22:05.584629 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-2ebdd804-c583-40b1-bd89-3bdaf0db037a tempest-ServerDiskConfigTestJSON-1882845406 tempest-ServerDiskConfigTestJSON-1882845406-project] Getting quotas for project 0e8e3faff3b247d89b96af3ff20a93c1. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 19 00:22:05.589308 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-2ebdd804-c583-40b1-bd89-3bdaf0db037a tempest-ServerDiskConfigTestJSON-1882845406 tempest-ServerDiskConfigTestJSON-1882845406-project] Getting quotas for user da0873c0156141f88fcbf1acbd0dc42d and project 0e8e3faff3b247d89b96af3ff20a93c1. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 19 00:22:05.600781 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-2ebdd804-c583-40b1-bd89-3bdaf0db037a tempest-ServerDiskConfigTestJSON-1882845406 tempest-ServerDiskConfigTestJSON-1882845406-project] [instance: f53bf688-3d08-4bb7-8408-a85f2fe9f536] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120658) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 19 00:22:05.601540 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-2ebdd804-c583-40b1-bd89-3bdaf0db037a tempest-ServerDiskConfigTestJSON-1882845406 tempest-ServerDiskConfigTestJSON-1882845406-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 00:22:05.601963 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-2ebdd804-c583-40b1-bd89-3bdaf0db037a tempest-ServerDiskConfigTestJSON-1882845406 tempest-ServerDiskConfigTestJSON-1882845406-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 00:22:05.602345 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-2ebdd804-c583-40b1-bd89-3bdaf0db037a tempest-ServerDiskConfigTestJSON-1882845406 tempest-ServerDiskConfigTestJSON-1882845406-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 00:22:05.694849 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-2ebdd804-c583-40b1-bd89-3bdaf0db037a tempest-ServerDiskConfigTestJSON-1882845406 tempest-ServerDiskConfigTestJSON-1882845406-project] [instance: f53bf688-3d08-4bb7-8408-a85f2fe9f536] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='87b0fec9-a948-41f9-a3c0-f0a6a5a8f1e7',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120658) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 19 00:22:05.696221 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-2ebdd804-c583-40b1-bd89-3bdaf0db037a tempest-ServerDiskConfigTestJSON-1882845406 tempest-ServerDiskConfigTestJSON-1882845406-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 00:22:05.697508 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-2ebdd804-c583-40b1-bd89-3bdaf0db037a tempest-ServerDiskConfigTestJSON-1882845406 tempest-ServerDiskConfigTestJSON-1882845406-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 00:22:05.697907 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-2ebdd804-c583-40b1-bd89-3bdaf0db037a tempest-ServerDiskConfigTestJSON-1882845406 tempest-ServerDiskConfigTestJSON-1882845406-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 00:22:05.941535 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-2ebdd804-c583-40b1-bd89-3bdaf0db037a tempest-ServerDiskConfigTestJSON-1882845406 tempest-ServerDiskConfigTestJSON-1882845406-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 00:22:05.941922 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-2ebdd804-c583-40b1-bd89-3bdaf0db037a tempest-ServerDiskConfigTestJSON-1882845406 tempest-ServerDiskConfigTestJSON-1882845406-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 00:22:05.942480 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-2ebdd804-c583-40b1-bd89-3bdaf0db037a tempest-ServerDiskConfigTestJSON-1882845406 tempest-ServerDiskConfigTestJSON-1882845406-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 00:22:05.942927 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-2ebdd804-c583-40b1-bd89-3bdaf0db037a tempest-ServerDiskConfigTestJSON-1882845406 tempest-ServerDiskConfigTestJSON-1882845406-project] Acquiring lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 19 00:22:05.943454 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-2ebdd804-c583-40b1-bd89-3bdaf0db037a tempest-ServerDiskConfigTestJSON-1882845406 tempest-ServerDiskConfigTestJSON-1882845406-project] Acquired lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 19 00:22:05.944216 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ed7cb068-11b1-42c4-8868-66d6934400b2 None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 00:22:05.944627 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ed7cb068-11b1-42c4-8868-66d6934400b2 None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 00:22:05.945010 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ed7cb068-11b1-42c4-8868-66d6934400b2 None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 00:22:05.945656 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ed7cb068-11b1-42c4-8868-66d6934400b2 None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 00:22:05.946061 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ed7cb068-11b1-42c4-8868-66d6934400b2 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 00:22:05.946465 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ed7cb068-11b1-42c4-8868-66d6934400b2 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 00:22:05.955427 n-d-853704-1 nova-conductor[120658]: ERROR nova.compute.rpcapi [None req-ed7cb068-11b1-42c4-8868-66d6934400b2 None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 19 00:22:05.955849 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ed7cb068-11b1-42c4-8868-66d6934400b2 None None] Releasing lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 19 00:22:05.956766 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server [None req-ed7cb068-11b1-42c4-8868-66d6934400b2 None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 19 00:22:05.956766 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 19 00:22:05.956766 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 19 00:22:05.956766 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 19 00:22:05.956766 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 19 00:22:05.956766 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 19 00:22:05.956766 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 19 00:22:05.956766 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 19 00:22:05.956766 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 19 00:22:05.956766 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 19 00:22:05.956766 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 19 00:22:05.956766 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 19 00:22:05.956766 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 19 00:22:05.956766 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 19 00:22:05.956766 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 19 00:22:05.956766 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 19 00:22:05.956766 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 19 00:22:05.956766 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 19 00:22:05.956766 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 19 00:22:05.956766 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 19 00:22:05.956766 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 19 00:22:05.956766 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 19 00:22:05.956766 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 19 00:22:05.956766 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 19 00:22:05.956766 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 19 00:22:15.857443 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=8 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:22:15.858245 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=16,nova_api:UPDATE=1,nova_api:DELETE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:22:15.952831 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:22:15.953502 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell1:SELECT=11,nova_cell1:SAVEPOINT=1,nova_cell1:INSERT=18,nova_cell1:RELEASE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:22:33.761941 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:22:43.742167 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=14,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:22:53.768231 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=3,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:23:23.733294 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:23:33.764430 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:23:35.535569 n-d-853704-1 nova-conductor[120657]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 19 00:23:53.739001 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=5,nova_cell0:UPDATE=3 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:23:53.772115 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=3,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:24:33.154124 n-d-853704-1 nova-conductor[120657]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 19 00:24:33.753742 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:24:33.773466 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:25:01.445209 n-d-853704-1 nova-conductor[120658]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 19 00:25:13.757457 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:25:13.777763 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:25:53.752187 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:25:53.773374 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:26:33.762208 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=7,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:26:33.782068 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=7,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:26:55.545604 n-d-853704-1 nova-conductor[120657]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 19 00:27:03.760692 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=6,nova_cell0:UPDATE=3 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:27:13.790357 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:27:43.768860 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:27:53.161566 n-d-853704-1 nova-conductor[120657]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 19 00:27:53.794625 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:28:06.151738 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-b47c74cd-111f-40c2-ac83-77df5653760b tempest-ServerDiskConfigTestJSON-1882845406 tempest-ServerDiskConfigTestJSON-1882845406-project] Took 0.45 seconds to select destinations for 1 instance(s). {{(pid=120657) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 19 00:28:06.172957 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-b47c74cd-111f-40c2-ac83-77df5653760b tempest-ServerDiskConfigTestJSON-1882845406 tempest-ServerDiskConfigTestJSON-1882845406-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 00:28:06.173395 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-b47c74cd-111f-40c2-ac83-77df5653760b tempest-ServerDiskConfigTestJSON-1882845406 tempest-ServerDiskConfigTestJSON-1882845406-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 00:28:06.173785 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-b47c74cd-111f-40c2-ac83-77df5653760b tempest-ServerDiskConfigTestJSON-1882845406 tempest-ServerDiskConfigTestJSON-1882845406-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 00:28:06.265812 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-b47c74cd-111f-40c2-ac83-77df5653760b tempest-ServerDiskConfigTestJSON-1882845406 tempest-ServerDiskConfigTestJSON-1882845406-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 00:28:06.266254 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-b47c74cd-111f-40c2-ac83-77df5653760b tempest-ServerDiskConfigTestJSON-1882845406 tempest-ServerDiskConfigTestJSON-1882845406-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 00:28:06.266641 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-b47c74cd-111f-40c2-ac83-77df5653760b tempest-ServerDiskConfigTestJSON-1882845406 tempest-ServerDiskConfigTestJSON-1882845406-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 00:28:06.267297 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-b47c74cd-111f-40c2-ac83-77df5653760b tempest-ServerDiskConfigTestJSON-1882845406 tempest-ServerDiskConfigTestJSON-1882845406-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 00:28:06.267726 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-b47c74cd-111f-40c2-ac83-77df5653760b tempest-ServerDiskConfigTestJSON-1882845406 tempest-ServerDiskConfigTestJSON-1882845406-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 00:28:06.268110 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-b47c74cd-111f-40c2-ac83-77df5653760b tempest-ServerDiskConfigTestJSON-1882845406 tempest-ServerDiskConfigTestJSON-1882845406-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 00:28:06.283210 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-b47c74cd-111f-40c2-ac83-77df5653760b tempest-ServerDiskConfigTestJSON-1882845406 tempest-ServerDiskConfigTestJSON-1882845406-project] Getting quotas for project 0e8e3faff3b247d89b96af3ff20a93c1. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 19 00:28:06.288797 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-b47c74cd-111f-40c2-ac83-77df5653760b tempest-ServerDiskConfigTestJSON-1882845406 tempest-ServerDiskConfigTestJSON-1882845406-project] Getting quotas for user da0873c0156141f88fcbf1acbd0dc42d and project 0e8e3faff3b247d89b96af3ff20a93c1. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 19 00:28:06.299409 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-b47c74cd-111f-40c2-ac83-77df5653760b tempest-ServerDiskConfigTestJSON-1882845406 tempest-ServerDiskConfigTestJSON-1882845406-project] [instance: 5545e673-50a3-47d0-a66a-6099d419d208] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120657) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 19 00:28:06.300257 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-b47c74cd-111f-40c2-ac83-77df5653760b tempest-ServerDiskConfigTestJSON-1882845406 tempest-ServerDiskConfigTestJSON-1882845406-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 00:28:06.301198 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-b47c74cd-111f-40c2-ac83-77df5653760b tempest-ServerDiskConfigTestJSON-1882845406 tempest-ServerDiskConfigTestJSON-1882845406-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 00:28:06.301596 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-b47c74cd-111f-40c2-ac83-77df5653760b tempest-ServerDiskConfigTestJSON-1882845406 tempest-ServerDiskConfigTestJSON-1882845406-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 00:28:06.339206 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-b47c74cd-111f-40c2-ac83-77df5653760b tempest-ServerDiskConfigTestJSON-1882845406 tempest-ServerDiskConfigTestJSON-1882845406-project] [instance: 5545e673-50a3-47d0-a66a-6099d419d208] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='87b0fec9-a948-41f9-a3c0-f0a6a5a8f1e7',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120657) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 19 00:28:06.340431 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-b47c74cd-111f-40c2-ac83-77df5653760b tempest-ServerDiskConfigTestJSON-1882845406 tempest-ServerDiskConfigTestJSON-1882845406-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 00:28:06.340910 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-b47c74cd-111f-40c2-ac83-77df5653760b tempest-ServerDiskConfigTestJSON-1882845406 tempest-ServerDiskConfigTestJSON-1882845406-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 00:28:06.341347 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-b47c74cd-111f-40c2-ac83-77df5653760b tempest-ServerDiskConfigTestJSON-1882845406 tempest-ServerDiskConfigTestJSON-1882845406-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 00:28:06.528595 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-b47c74cd-111f-40c2-ac83-77df5653760b tempest-ServerDiskConfigTestJSON-1882845406 tempest-ServerDiskConfigTestJSON-1882845406-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 00:28:06.529029 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-b47c74cd-111f-40c2-ac83-77df5653760b tempest-ServerDiskConfigTestJSON-1882845406 tempest-ServerDiskConfigTestJSON-1882845406-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 00:28:06.529429 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-b47c74cd-111f-40c2-ac83-77df5653760b tempest-ServerDiskConfigTestJSON-1882845406 tempest-ServerDiskConfigTestJSON-1882845406-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 00:28:06.529882 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-b47c74cd-111f-40c2-ac83-77df5653760b tempest-ServerDiskConfigTestJSON-1882845406 tempest-ServerDiskConfigTestJSON-1882845406-project] Acquiring lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 19 00:28:06.530230 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-b47c74cd-111f-40c2-ac83-77df5653760b tempest-ServerDiskConfigTestJSON-1882845406 tempest-ServerDiskConfigTestJSON-1882845406-project] Acquired lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 19 00:28:06.530959 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-884701cf-2745-4214-b953-fbdd96d21fdc None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 00:28:06.531372 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-884701cf-2745-4214-b953-fbdd96d21fdc None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 00:28:06.531776 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-884701cf-2745-4214-b953-fbdd96d21fdc None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 00:28:06.532478 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-884701cf-2745-4214-b953-fbdd96d21fdc None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 00:28:06.532887 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-884701cf-2745-4214-b953-fbdd96d21fdc None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 00:28:06.533263 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-884701cf-2745-4214-b953-fbdd96d21fdc None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 00:28:06.542986 n-d-853704-1 nova-conductor[120657]: ERROR nova.compute.rpcapi [None req-884701cf-2745-4214-b953-fbdd96d21fdc None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 19 00:28:06.543373 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-884701cf-2745-4214-b953-fbdd96d21fdc None None] Releasing lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 19 00:28:06.544373 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server [None req-884701cf-2745-4214-b953-fbdd96d21fdc None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 19 00:28:06.544373 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 19 00:28:06.544373 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 19 00:28:06.544373 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 19 00:28:06.544373 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 19 00:28:06.544373 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 19 00:28:06.544373 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 19 00:28:06.544373 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 19 00:28:06.544373 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 19 00:28:06.544373 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 19 00:28:06.544373 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 19 00:28:06.544373 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 19 00:28:06.544373 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 19 00:28:06.544373 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 19 00:28:06.544373 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 19 00:28:06.544373 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 19 00:28:06.544373 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 19 00:28:06.544373 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 19 00:28:06.544373 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 19 00:28:06.544373 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 19 00:28:06.544373 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 19 00:28:06.544373 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 19 00:28:06.544373 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 19 00:28:06.544373 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 19 00:28:06.544373 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 19 00:28:16.462025 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=12 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:28:16.462902 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=12,nova_api:UPDATE=1,nova_api:DELETE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:28:16.540474 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:28:16.541128 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell1:SELECT=11,nova_cell1:INSERT=17 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:28:21.453134 n-d-853704-1 nova-conductor[120658]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 19 00:28:23.766884 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:28:33.790861 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=14,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:29:03.774883 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=7,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:29:03.794422 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=6,nova_cell0:UPDATE=3 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:29:23.783422 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=4,nova_cell0:UPDATE=2 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:29:43.792094 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:30:03.782595 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:30:15.553959 n-d-853704-1 nova-conductor[120657]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 19 00:30:23.797891 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:30:43.785295 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:31:03.791824 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=3,nova_cell0:UPDATE=2 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:31:03.800511 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=7,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:31:13.169367 n-d-853704-1 nova-conductor[120657]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 19 00:31:41.473618 n-d-853704-1 nova-conductor[120658]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 19 00:31:43.792773 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:31:43.803480 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:32:03.802051 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=4,nova_cell0:UPDATE=2 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:32:03.811466 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=4,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:32:33.818166 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=6,nova_cell0:UPDATE=3 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:32:43.811586 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:33:03.818806 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=6,nova_cell0:UPDATE=3 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:33:23.820897 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:33:35.562091 n-d-853704-1 nova-conductor[120657]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 19 00:33:43.817831 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:34:03.821941 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:34:23.190847 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-eb3929f0-f280-4444-a939-9cfbf43bac7f tempest-ServerActionsTestJSON-995662025 tempest-ServerActionsTestJSON-995662025-project] Took 0.72 seconds to select destinations for 1 instance(s). {{(pid=120658) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 19 00:34:23.211324 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-eb3929f0-f280-4444-a939-9cfbf43bac7f tempest-ServerActionsTestJSON-995662025 tempest-ServerActionsTestJSON-995662025-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 00:34:23.211798 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-eb3929f0-f280-4444-a939-9cfbf43bac7f tempest-ServerActionsTestJSON-995662025 tempest-ServerActionsTestJSON-995662025-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 00:34:23.212196 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-eb3929f0-f280-4444-a939-9cfbf43bac7f tempest-ServerActionsTestJSON-995662025 tempest-ServerActionsTestJSON-995662025-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 00:34:23.372516 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-eb3929f0-f280-4444-a939-9cfbf43bac7f tempest-ServerActionsTestJSON-995662025 tempest-ServerActionsTestJSON-995662025-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 00:34:23.372954 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-eb3929f0-f280-4444-a939-9cfbf43bac7f tempest-ServerActionsTestJSON-995662025 tempest-ServerActionsTestJSON-995662025-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 00:34:23.373341 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-eb3929f0-f280-4444-a939-9cfbf43bac7f tempest-ServerActionsTestJSON-995662025 tempest-ServerActionsTestJSON-995662025-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 00:34:23.373991 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-eb3929f0-f280-4444-a939-9cfbf43bac7f tempest-ServerActionsTestJSON-995662025 tempest-ServerActionsTestJSON-995662025-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 00:34:23.374394 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-eb3929f0-f280-4444-a939-9cfbf43bac7f tempest-ServerActionsTestJSON-995662025 tempest-ServerActionsTestJSON-995662025-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 00:34:23.374771 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-eb3929f0-f280-4444-a939-9cfbf43bac7f tempest-ServerActionsTestJSON-995662025 tempest-ServerActionsTestJSON-995662025-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 00:34:23.390296 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-eb3929f0-f280-4444-a939-9cfbf43bac7f tempest-ServerActionsTestJSON-995662025 tempest-ServerActionsTestJSON-995662025-project] Getting quotas for project 387eb80598954c31bf38255488a89715. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 19 00:34:23.394621 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-eb3929f0-f280-4444-a939-9cfbf43bac7f tempest-ServerActionsTestJSON-995662025 tempest-ServerActionsTestJSON-995662025-project] Getting quotas for user fd8cfa44fa04488f86b6f95d7a5981e5 and project 387eb80598954c31bf38255488a89715. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 19 00:34:23.406505 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-eb3929f0-f280-4444-a939-9cfbf43bac7f tempest-ServerActionsTestJSON-995662025 tempest-ServerActionsTestJSON-995662025-project] [instance: d38f6910-a456-49d9-aac8-8b19bac80c3a] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120658) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 19 00:34:23.407287 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-eb3929f0-f280-4444-a939-9cfbf43bac7f tempest-ServerActionsTestJSON-995662025 tempest-ServerActionsTestJSON-995662025-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 00:34:23.407857 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-eb3929f0-f280-4444-a939-9cfbf43bac7f tempest-ServerActionsTestJSON-995662025 tempest-ServerActionsTestJSON-995662025-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 00:34:23.408240 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-eb3929f0-f280-4444-a939-9cfbf43bac7f tempest-ServerActionsTestJSON-995662025 tempest-ServerActionsTestJSON-995662025-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 00:34:23.463956 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-eb3929f0-f280-4444-a939-9cfbf43bac7f tempest-ServerActionsTestJSON-995662025 tempest-ServerActionsTestJSON-995662025-project] [instance: d38f6910-a456-49d9-aac8-8b19bac80c3a] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='87b0fec9-a948-41f9-a3c0-f0a6a5a8f1e7',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120658) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 19 00:34:23.465128 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-eb3929f0-f280-4444-a939-9cfbf43bac7f tempest-ServerActionsTestJSON-995662025 tempest-ServerActionsTestJSON-995662025-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 00:34:23.465556 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-eb3929f0-f280-4444-a939-9cfbf43bac7f tempest-ServerActionsTestJSON-995662025 tempest-ServerActionsTestJSON-995662025-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 00:34:23.465953 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-eb3929f0-f280-4444-a939-9cfbf43bac7f tempest-ServerActionsTestJSON-995662025 tempest-ServerActionsTestJSON-995662025-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 00:34:23.698949 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-eb3929f0-f280-4444-a939-9cfbf43bac7f tempest-ServerActionsTestJSON-995662025 tempest-ServerActionsTestJSON-995662025-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 00:34:23.699483 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-eb3929f0-f280-4444-a939-9cfbf43bac7f tempest-ServerActionsTestJSON-995662025 tempest-ServerActionsTestJSON-995662025-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 00:34:23.699927 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-eb3929f0-f280-4444-a939-9cfbf43bac7f tempest-ServerActionsTestJSON-995662025 tempest-ServerActionsTestJSON-995662025-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 00:34:23.700404 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-eb3929f0-f280-4444-a939-9cfbf43bac7f tempest-ServerActionsTestJSON-995662025 tempest-ServerActionsTestJSON-995662025-project] Acquiring lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 19 00:34:23.700758 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-eb3929f0-f280-4444-a939-9cfbf43bac7f tempest-ServerActionsTestJSON-995662025 tempest-ServerActionsTestJSON-995662025-project] Acquired lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 19 00:34:23.701486 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ed9fb7f7-fda5-445c-ad58-f35caf1669cf None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 00:34:23.702014 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ed9fb7f7-fda5-445c-ad58-f35caf1669cf None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 00:34:23.702359 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ed9fb7f7-fda5-445c-ad58-f35caf1669cf None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 00:34:23.702996 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ed9fb7f7-fda5-445c-ad58-f35caf1669cf None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 00:34:23.703425 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ed9fb7f7-fda5-445c-ad58-f35caf1669cf None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 00:34:23.703830 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ed9fb7f7-fda5-445c-ad58-f35caf1669cf None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 00:34:23.713080 n-d-853704-1 nova-conductor[120658]: ERROR nova.compute.rpcapi [None req-ed9fb7f7-fda5-445c-ad58-f35caf1669cf None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 19 00:34:23.713524 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-ed9fb7f7-fda5-445c-ad58-f35caf1669cf None None] Releasing lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 19 00:34:23.714439 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server [None req-ed9fb7f7-fda5-445c-ad58-f35caf1669cf None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 19 00:34:23.714439 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 19 00:34:23.714439 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 19 00:34:23.714439 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 19 00:34:23.714439 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 19 00:34:23.714439 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 19 00:34:23.714439 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 19 00:34:23.714439 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 19 00:34:23.714439 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 19 00:34:23.714439 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 19 00:34:23.714439 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 19 00:34:23.714439 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 19 00:34:23.714439 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 19 00:34:23.714439 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 19 00:34:23.714439 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 19 00:34:23.714439 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 19 00:34:23.714439 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 19 00:34:23.714439 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 19 00:34:23.714439 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 19 00:34:23.714439 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 19 00:34:23.714439 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 19 00:34:23.714439 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 19 00:34:23.714439 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 19 00:34:23.714439 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 19 00:34:23.714439 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 19 00:34:23.826475 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=7,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:34:33.178169 n-d-853704-1 nova-conductor[120657]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 19 00:34:33.604166 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=13,nova_api:UPDATE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:34:33.604947 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=11,nova_api:DELETE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:34:33.709712 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:34:33.710276 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell1:SELECT=11,nova_cell1:SAVEPOINT=1,nova_cell1:INSERT=18,nova_cell1:RELEASE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:34:43.822351 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=14,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:35:03.827828 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:35:03.828347 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=3,nova_cell0:UPDATE=2 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:35:43.824273 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:35:43.835463 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:36:13.833049 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=5,nova_cell0:UPDATE=3 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:36:13.843511 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=5,nova_cell0:UPDATE=3 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:36:33.843980 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=4,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:36:33.851847 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=4,nova_cell0:UPDATE=2 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:36:55.570330 n-d-853704-1 nova-conductor[120657]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 19 00:37:13.845474 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:37:13.855412 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:37:53.188112 n-d-853704-1 nova-conductor[120657]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 19 00:37:53.846533 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:37:53.855847 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:38:23.851803 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=6,nova_cell0:UPDATE=3 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:38:23.861606 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=6,nova_cell0:UPDATE=3 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:38:53.854085 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=6,nova_cell0:UPDATE=3 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:39:03.869841 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:39:23.865412 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=6,nova_cell0:UPDATE=3 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:39:23.874400 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=4,nova_cell0:UPDATE=2 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:39:53.870313 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=6,nova_cell0:UPDATE=3 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:39:53.880902 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=6,nova_cell0:UPDATE=3 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:40:01.486220 n-d-853704-1 nova-conductor[120658]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 19 00:40:15.577953 n-d-853704-1 nova-conductor[120657]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 19 00:40:23.877247 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=6,nova_cell0:UPDATE=3 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:40:33.877220 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:40:36.019230 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-b553bcd2-2bc2-4703-925a-0d9b957ec750 tempest-ServersNegativeTestJSON-1159809409 tempest-ServersNegativeTestJSON-1159809409-project] Took 0.72 seconds to select destinations for 1 instance(s). {{(pid=120657) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 19 00:40:36.039836 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-b553bcd2-2bc2-4703-925a-0d9b957ec750 tempest-ServersNegativeTestJSON-1159809409 tempest-ServersNegativeTestJSON-1159809409-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 00:40:36.040284 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-b553bcd2-2bc2-4703-925a-0d9b957ec750 tempest-ServersNegativeTestJSON-1159809409 tempest-ServersNegativeTestJSON-1159809409-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 00:40:36.040688 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-b553bcd2-2bc2-4703-925a-0d9b957ec750 tempest-ServersNegativeTestJSON-1159809409 tempest-ServersNegativeTestJSON-1159809409-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 00:40:36.258567 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-b553bcd2-2bc2-4703-925a-0d9b957ec750 tempest-ServersNegativeTestJSON-1159809409 tempest-ServersNegativeTestJSON-1159809409-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 00:40:36.259037 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-b553bcd2-2bc2-4703-925a-0d9b957ec750 tempest-ServersNegativeTestJSON-1159809409 tempest-ServersNegativeTestJSON-1159809409-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 00:40:36.259595 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-b553bcd2-2bc2-4703-925a-0d9b957ec750 tempest-ServersNegativeTestJSON-1159809409 tempest-ServersNegativeTestJSON-1159809409-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 00:40:36.260307 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-b553bcd2-2bc2-4703-925a-0d9b957ec750 tempest-ServersNegativeTestJSON-1159809409 tempest-ServersNegativeTestJSON-1159809409-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 00:40:36.260750 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-b553bcd2-2bc2-4703-925a-0d9b957ec750 tempest-ServersNegativeTestJSON-1159809409 tempest-ServersNegativeTestJSON-1159809409-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 00:40:36.261136 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-b553bcd2-2bc2-4703-925a-0d9b957ec750 tempest-ServersNegativeTestJSON-1159809409 tempest-ServersNegativeTestJSON-1159809409-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 00:40:36.277420 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-b553bcd2-2bc2-4703-925a-0d9b957ec750 tempest-ServersNegativeTestJSON-1159809409 tempest-ServersNegativeTestJSON-1159809409-project] Getting quotas for project 5c8dd446ac8f4dff87cab6c04b9f5bb5. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 19 00:40:36.282035 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-b553bcd2-2bc2-4703-925a-0d9b957ec750 tempest-ServersNegativeTestJSON-1159809409 tempest-ServersNegativeTestJSON-1159809409-project] Getting quotas for user be63e1ca47234a389bd18be8017456b1 and project 5c8dd446ac8f4dff87cab6c04b9f5bb5. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 19 00:40:36.293221 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-b553bcd2-2bc2-4703-925a-0d9b957ec750 tempest-ServersNegativeTestJSON-1159809409 tempest-ServersNegativeTestJSON-1159809409-project] [instance: e0d96eaa-2c2d-4ccf-ba57-c2a56135d99c] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120657) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 19 00:40:36.293996 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-b553bcd2-2bc2-4703-925a-0d9b957ec750 tempest-ServersNegativeTestJSON-1159809409 tempest-ServersNegativeTestJSON-1159809409-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 00:40:36.294433 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-b553bcd2-2bc2-4703-925a-0d9b957ec750 tempest-ServersNegativeTestJSON-1159809409 tempest-ServersNegativeTestJSON-1159809409-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 00:40:36.294832 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-b553bcd2-2bc2-4703-925a-0d9b957ec750 tempest-ServersNegativeTestJSON-1159809409 tempest-ServersNegativeTestJSON-1159809409-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 00:40:36.356361 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-b553bcd2-2bc2-4703-925a-0d9b957ec750 tempest-ServersNegativeTestJSON-1159809409 tempest-ServersNegativeTestJSON-1159809409-project] [instance: e0d96eaa-2c2d-4ccf-ba57-c2a56135d99c] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='87b0fec9-a948-41f9-a3c0-f0a6a5a8f1e7',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120657) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 19 00:40:36.357502 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-b553bcd2-2bc2-4703-925a-0d9b957ec750 tempest-ServersNegativeTestJSON-1159809409 tempest-ServersNegativeTestJSON-1159809409-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 00:40:36.357939 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-b553bcd2-2bc2-4703-925a-0d9b957ec750 tempest-ServersNegativeTestJSON-1159809409 tempest-ServersNegativeTestJSON-1159809409-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 00:40:36.358435 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-b553bcd2-2bc2-4703-925a-0d9b957ec750 tempest-ServersNegativeTestJSON-1159809409 tempest-ServersNegativeTestJSON-1159809409-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 00:40:36.615126 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-b553bcd2-2bc2-4703-925a-0d9b957ec750 tempest-ServersNegativeTestJSON-1159809409 tempest-ServersNegativeTestJSON-1159809409-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 00:40:36.615646 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-b553bcd2-2bc2-4703-925a-0d9b957ec750 tempest-ServersNegativeTestJSON-1159809409 tempest-ServersNegativeTestJSON-1159809409-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 00:40:36.616144 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-b553bcd2-2bc2-4703-925a-0d9b957ec750 tempest-ServersNegativeTestJSON-1159809409 tempest-ServersNegativeTestJSON-1159809409-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 00:40:36.616679 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-b553bcd2-2bc2-4703-925a-0d9b957ec750 tempest-ServersNegativeTestJSON-1159809409 tempest-ServersNegativeTestJSON-1159809409-project] Acquiring lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 19 00:40:36.617189 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-b553bcd2-2bc2-4703-925a-0d9b957ec750 tempest-ServersNegativeTestJSON-1159809409 tempest-ServersNegativeTestJSON-1159809409-project] Acquired lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 19 00:40:36.618227 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c37e4899-4f3a-480e-ac19-f34c2e9a6478 None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 00:40:36.619598 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c37e4899-4f3a-480e-ac19-f34c2e9a6478 None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 00:40:36.623337 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c37e4899-4f3a-480e-ac19-f34c2e9a6478 None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 00:40:36.623337 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c37e4899-4f3a-480e-ac19-f34c2e9a6478 None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 00:40:36.623337 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c37e4899-4f3a-480e-ac19-f34c2e9a6478 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 00:40:36.623337 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c37e4899-4f3a-480e-ac19-f34c2e9a6478 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 00:40:36.631512 n-d-853704-1 nova-conductor[120657]: ERROR nova.compute.rpcapi [None req-c37e4899-4f3a-480e-ac19-f34c2e9a6478 None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 19 00:40:36.631924 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c37e4899-4f3a-480e-ac19-f34c2e9a6478 None None] Releasing lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 19 00:40:36.632849 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server [None req-c37e4899-4f3a-480e-ac19-f34c2e9a6478 None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 19 00:40:36.632849 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 19 00:40:36.632849 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 19 00:40:36.632849 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 19 00:40:36.632849 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 19 00:40:36.632849 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 19 00:40:36.632849 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 19 00:40:36.632849 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 19 00:40:36.632849 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 19 00:40:36.632849 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 19 00:40:36.632849 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 19 00:40:36.632849 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 19 00:40:36.632849 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 19 00:40:36.632849 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 19 00:40:36.632849 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 19 00:40:36.632849 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 19 00:40:36.632849 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 19 00:40:36.632849 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 19 00:40:36.632849 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 19 00:40:36.632849 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 19 00:40:36.632849 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 19 00:40:36.632849 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 19 00:40:36.632849 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 19 00:40:36.632849 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 19 00:40:36.632849 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 19 00:40:46.430788 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=12 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:40:46.542197 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=12,nova_api:UPDATE=1,nova_api:DELETE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:40:46.627536 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:40:46.628640 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell1:SELECT=11,nova_cell1:SAVEPOINT=1,nova_cell1:INSERT=18,nova_cell1:RELEASE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:40:53.884504 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=3,nova_cell0:UPDATE=2 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:40:53.885710 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=12,nova_cell0:UPDATE=3 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:41:33.877230 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:41:33.887040 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:41:53.889031 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=3,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:42:03.881536 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=5,nova_cell0:UPDATE=3 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:42:33.893320 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:42:43.883243 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:42:53.201544 n-d-853704-1 nova-conductor[120657]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 19 00:42:53.898774 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=3,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:43:13.895027 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=5,nova_cell0:UPDATE=3 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:43:13.906304 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=4,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:43:21.494548 n-d-853704-1 nova-conductor[120658]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 19 00:43:35.586681 n-d-853704-1 nova-conductor[120657]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 19 00:43:53.901425 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:43:53.911738 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:44:33.902561 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:44:33.915142 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:44:53.915369 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=4,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:45:13.905314 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:45:23.920519 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=6,nova_cell0:UPDATE=3 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:45:43.905331 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=5,nova_cell0:UPDATE=3 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:45:43.925852 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=4,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:46:13.209870 n-d-853704-1 nova-conductor[120657]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 19 00:46:23.911423 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:46:23.932837 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:46:41.502437 n-d-853704-1 nova-conductor[120658]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 19 00:46:43.919831 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=3,nova_cell0:UPDATE=2 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:46:53.944562 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=5,nova_cell0:UPDATE=3 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:46:55.594007 n-d-853704-1 nova-conductor[120657]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 19 00:47:22.799353 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-1a2bbf1f-93c2-4836-933e-0709073c2f17 tempest-TestEncryptedCinderVolumes-1861635015 tempest-TestEncryptedCinderVolumes-1861635015-project] Took 0.61 seconds to select destinations for 1 instance(s). {{(pid=120658) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 19 00:47:22.824676 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-1a2bbf1f-93c2-4836-933e-0709073c2f17 tempest-TestEncryptedCinderVolumes-1861635015 tempest-TestEncryptedCinderVolumes-1861635015-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 00:47:22.825112 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-1a2bbf1f-93c2-4836-933e-0709073c2f17 tempest-TestEncryptedCinderVolumes-1861635015 tempest-TestEncryptedCinderVolumes-1861635015-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 00:47:22.825501 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-1a2bbf1f-93c2-4836-933e-0709073c2f17 tempest-TestEncryptedCinderVolumes-1861635015 tempest-TestEncryptedCinderVolumes-1861635015-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 00:47:23.067627 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:47:23.075447 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-1a2bbf1f-93c2-4836-933e-0709073c2f17 tempest-TestEncryptedCinderVolumes-1861635015 tempest-TestEncryptedCinderVolumes-1861635015-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 00:47:23.076925 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-1a2bbf1f-93c2-4836-933e-0709073c2f17 tempest-TestEncryptedCinderVolumes-1861635015 tempest-TestEncryptedCinderVolumes-1861635015-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 00:47:23.080306 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-1a2bbf1f-93c2-4836-933e-0709073c2f17 tempest-TestEncryptedCinderVolumes-1861635015 tempest-TestEncryptedCinderVolumes-1861635015-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 00:47:23.080306 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-1a2bbf1f-93c2-4836-933e-0709073c2f17 tempest-TestEncryptedCinderVolumes-1861635015 tempest-TestEncryptedCinderVolumes-1861635015-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 00:47:23.080306 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-1a2bbf1f-93c2-4836-933e-0709073c2f17 tempest-TestEncryptedCinderVolumes-1861635015 tempest-TestEncryptedCinderVolumes-1861635015-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 00:47:23.080306 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-1a2bbf1f-93c2-4836-933e-0709073c2f17 tempest-TestEncryptedCinderVolumes-1861635015 tempest-TestEncryptedCinderVolumes-1861635015-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 00:47:23.102184 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-1a2bbf1f-93c2-4836-933e-0709073c2f17 tempest-TestEncryptedCinderVolumes-1861635015 tempest-TestEncryptedCinderVolumes-1861635015-project] Getting quotas for project 9a2442d1e525437f9924a36c183502dc. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 19 00:47:23.119479 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-1a2bbf1f-93c2-4836-933e-0709073c2f17 tempest-TestEncryptedCinderVolumes-1861635015 tempest-TestEncryptedCinderVolumes-1861635015-project] Getting quotas for user 1588807348bf4fc2b0482241e732737e and project 9a2442d1e525437f9924a36c183502dc. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 19 00:47:23.134530 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-1a2bbf1f-93c2-4836-933e-0709073c2f17 tempest-TestEncryptedCinderVolumes-1861635015 tempest-TestEncryptedCinderVolumes-1861635015-project] [instance: 617ea25b-24e6-45e2-aba0-21875778ea02] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120658) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 19 00:47:23.135308 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-1a2bbf1f-93c2-4836-933e-0709073c2f17 tempest-TestEncryptedCinderVolumes-1861635015 tempest-TestEncryptedCinderVolumes-1861635015-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 00:47:23.135801 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-1a2bbf1f-93c2-4836-933e-0709073c2f17 tempest-TestEncryptedCinderVolumes-1861635015 tempest-TestEncryptedCinderVolumes-1861635015-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 00:47:23.137328 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-1a2bbf1f-93c2-4836-933e-0709073c2f17 tempest-TestEncryptedCinderVolumes-1861635015 tempest-TestEncryptedCinderVolumes-1861635015-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.002s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 00:47:23.210182 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-1a2bbf1f-93c2-4836-933e-0709073c2f17 tempest-TestEncryptedCinderVolumes-1861635015 tempest-TestEncryptedCinderVolumes-1861635015-project] [instance: 617ea25b-24e6-45e2-aba0-21875778ea02] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='87b0fec9-a948-41f9-a3c0-f0a6a5a8f1e7',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120658) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 19 00:47:23.211655 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-1a2bbf1f-93c2-4836-933e-0709073c2f17 tempest-TestEncryptedCinderVolumes-1861635015 tempest-TestEncryptedCinderVolumes-1861635015-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 00:47:23.212208 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-1a2bbf1f-93c2-4836-933e-0709073c2f17 tempest-TestEncryptedCinderVolumes-1861635015 tempest-TestEncryptedCinderVolumes-1861635015-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 00:47:23.212612 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-1a2bbf1f-93c2-4836-933e-0709073c2f17 tempest-TestEncryptedCinderVolumes-1861635015 tempest-TestEncryptedCinderVolumes-1861635015-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 00:47:23.484633 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-1a2bbf1f-93c2-4836-933e-0709073c2f17 tempest-TestEncryptedCinderVolumes-1861635015 tempest-TestEncryptedCinderVolumes-1861635015-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 00:47:23.485082 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-1a2bbf1f-93c2-4836-933e-0709073c2f17 tempest-TestEncryptedCinderVolumes-1861635015 tempest-TestEncryptedCinderVolumes-1861635015-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 00:47:23.485472 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-1a2bbf1f-93c2-4836-933e-0709073c2f17 tempest-TestEncryptedCinderVolumes-1861635015 tempest-TestEncryptedCinderVolumes-1861635015-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 00:47:23.485929 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-1a2bbf1f-93c2-4836-933e-0709073c2f17 tempest-TestEncryptedCinderVolumes-1861635015 tempest-TestEncryptedCinderVolumes-1861635015-project] Acquiring lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 19 00:47:23.486281 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-1a2bbf1f-93c2-4836-933e-0709073c2f17 tempest-TestEncryptedCinderVolumes-1861635015 tempest-TestEncryptedCinderVolumes-1861635015-project] Acquired lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 19 00:47:23.487039 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-cba55dcc-c729-433d-8787-9ddf7b7cb3be None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 00:47:23.487450 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-cba55dcc-c729-433d-8787-9ddf7b7cb3be None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 00:47:23.487878 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-cba55dcc-c729-433d-8787-9ddf7b7cb3be None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 00:47:23.488615 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-cba55dcc-c729-433d-8787-9ddf7b7cb3be None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 00:47:23.488948 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-cba55dcc-c729-433d-8787-9ddf7b7cb3be None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 00:47:23.489327 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-cba55dcc-c729-433d-8787-9ddf7b7cb3be None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 00:47:23.497609 n-d-853704-1 nova-conductor[120658]: ERROR nova.compute.rpcapi [None req-cba55dcc-c729-433d-8787-9ddf7b7cb3be None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 19 00:47:23.498002 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-cba55dcc-c729-433d-8787-9ddf7b7cb3be None None] Releasing lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 19 00:47:23.498907 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server [None req-cba55dcc-c729-433d-8787-9ddf7b7cb3be None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 19 00:47:23.498907 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 19 00:47:23.498907 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 19 00:47:23.498907 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 19 00:47:23.498907 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 19 00:47:23.498907 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 19 00:47:23.498907 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 19 00:47:23.498907 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 19 00:47:23.498907 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 19 00:47:23.498907 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 19 00:47:23.498907 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 19 00:47:23.498907 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 19 00:47:23.498907 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 19 00:47:23.498907 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 19 00:47:23.498907 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 19 00:47:23.498907 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 19 00:47:23.498907 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 19 00:47:23.498907 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 19 00:47:23.498907 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 19 00:47:23.498907 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 19 00:47:23.498907 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 19 00:47:23.498907 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 19 00:47:23.498907 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 19 00:47:23.498907 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 19 00:47:23.498907 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 19 00:47:23.947566 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=6,nova_cell0:UPDATE=3 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:47:33.307293 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=12 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:47:33.420051 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=12,nova_api:UPDATE=1,nova_api:DELETE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:47:33.496135 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:47:33.496784 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell1:SELECT=11,nova_cell1:SAVEPOINT=1,nova_cell1:INSERT=18,nova_cell1:RELEASE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:47:43.933248 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=2 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:47:43.953767 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=4,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:48:23.932359 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:48:23.953714 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:48:43.960329 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=3,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:49:03.935208 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:49:23.957933 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:49:33.218220 n-d-853704-1 nova-conductor[120657]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 19 00:49:33.941121 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=5,nova_cell0:UPDATE=3 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:49:43.965879 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=3,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:50:01.513614 n-d-853704-1 nova-conductor[120658]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 19 00:50:03.970943 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=4,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:50:13.950087 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:50:15.602464 n-d-853704-1 nova-conductor[120657]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 19 00:50:43.978042 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:50:53.953890 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:51:13.958973 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=3,nova_cell0:UPDATE=2 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:51:13.980893 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=6,nova_cell0:UPDATE=3 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:51:53.965610 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:51:53.985212 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:52:23.970051 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=6,nova_cell0:UPDATE=3 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:52:23.990857 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=6,nova_cell0:UPDATE=3 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:52:53.226153 n-d-853704-1 nova-conductor[120657]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 19 00:53:03.975263 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:53:04.003885 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:53:21.519187 n-d-853704-1 nova-conductor[120658]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 19 00:53:24.346790 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-fb963736-f191-4619-8e63-95dd69607b03 tempest-TestEncryptedCinderVolumes-1861635015 tempest-TestEncryptedCinderVolumes-1861635015-project] Took 0.46 seconds to select destinations for 1 instance(s). {{(pid=120657) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 19 00:53:24.367220 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-fb963736-f191-4619-8e63-95dd69607b03 tempest-TestEncryptedCinderVolumes-1861635015 tempest-TestEncryptedCinderVolumes-1861635015-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 00:53:24.367674 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-fb963736-f191-4619-8e63-95dd69607b03 tempest-TestEncryptedCinderVolumes-1861635015 tempest-TestEncryptedCinderVolumes-1861635015-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 00:53:24.368093 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-fb963736-f191-4619-8e63-95dd69607b03 tempest-TestEncryptedCinderVolumes-1861635015 tempest-TestEncryptedCinderVolumes-1861635015-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 00:53:24.493419 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-fb963736-f191-4619-8e63-95dd69607b03 tempest-TestEncryptedCinderVolumes-1861635015 tempest-TestEncryptedCinderVolumes-1861635015-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 00:53:24.493850 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-fb963736-f191-4619-8e63-95dd69607b03 tempest-TestEncryptedCinderVolumes-1861635015 tempest-TestEncryptedCinderVolumes-1861635015-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 00:53:24.494241 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-fb963736-f191-4619-8e63-95dd69607b03 tempest-TestEncryptedCinderVolumes-1861635015 tempest-TestEncryptedCinderVolumes-1861635015-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 00:53:24.494900 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-fb963736-f191-4619-8e63-95dd69607b03 tempest-TestEncryptedCinderVolumes-1861635015 tempest-TestEncryptedCinderVolumes-1861635015-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 00:53:24.495345 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-fb963736-f191-4619-8e63-95dd69607b03 tempest-TestEncryptedCinderVolumes-1861635015 tempest-TestEncryptedCinderVolumes-1861635015-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 00:53:24.495810 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-fb963736-f191-4619-8e63-95dd69607b03 tempest-TestEncryptedCinderVolumes-1861635015 tempest-TestEncryptedCinderVolumes-1861635015-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 00:53:24.511840 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-fb963736-f191-4619-8e63-95dd69607b03 tempest-TestEncryptedCinderVolumes-1861635015 tempest-TestEncryptedCinderVolumes-1861635015-project] Getting quotas for project 9a2442d1e525437f9924a36c183502dc. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 19 00:53:24.516386 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-fb963736-f191-4619-8e63-95dd69607b03 tempest-TestEncryptedCinderVolumes-1861635015 tempest-TestEncryptedCinderVolumes-1861635015-project] Getting quotas for user 1588807348bf4fc2b0482241e732737e and project 9a2442d1e525437f9924a36c183502dc. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 19 00:53:24.527705 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-fb963736-f191-4619-8e63-95dd69607b03 tempest-TestEncryptedCinderVolumes-1861635015 tempest-TestEncryptedCinderVolumes-1861635015-project] [instance: 354fdf8a-d382-4404-ac26-3ca5079d8708] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120657) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 19 00:53:24.528460 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-fb963736-f191-4619-8e63-95dd69607b03 tempest-TestEncryptedCinderVolumes-1861635015 tempest-TestEncryptedCinderVolumes-1861635015-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 00:53:24.528884 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-fb963736-f191-4619-8e63-95dd69607b03 tempest-TestEncryptedCinderVolumes-1861635015 tempest-TestEncryptedCinderVolumes-1861635015-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 00:53:24.529270 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-fb963736-f191-4619-8e63-95dd69607b03 tempest-TestEncryptedCinderVolumes-1861635015 tempest-TestEncryptedCinderVolumes-1861635015-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 00:53:24.596765 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-fb963736-f191-4619-8e63-95dd69607b03 tempest-TestEncryptedCinderVolumes-1861635015 tempest-TestEncryptedCinderVolumes-1861635015-project] [instance: 354fdf8a-d382-4404-ac26-3ca5079d8708] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='87b0fec9-a948-41f9-a3c0-f0a6a5a8f1e7',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120657) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 19 00:53:24.598010 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-fb963736-f191-4619-8e63-95dd69607b03 tempest-TestEncryptedCinderVolumes-1861635015 tempest-TestEncryptedCinderVolumes-1861635015-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 00:53:24.598456 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-fb963736-f191-4619-8e63-95dd69607b03 tempest-TestEncryptedCinderVolumes-1861635015 tempest-TestEncryptedCinderVolumes-1861635015-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 00:53:24.598850 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-fb963736-f191-4619-8e63-95dd69607b03 tempest-TestEncryptedCinderVolumes-1861635015 tempest-TestEncryptedCinderVolumes-1861635015-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 00:53:24.833349 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-fb963736-f191-4619-8e63-95dd69607b03 tempest-TestEncryptedCinderVolumes-1861635015 tempest-TestEncryptedCinderVolumes-1861635015-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 00:53:24.833916 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-fb963736-f191-4619-8e63-95dd69607b03 tempest-TestEncryptedCinderVolumes-1861635015 tempest-TestEncryptedCinderVolumes-1861635015-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 00:53:24.834424 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-fb963736-f191-4619-8e63-95dd69607b03 tempest-TestEncryptedCinderVolumes-1861635015 tempest-TestEncryptedCinderVolumes-1861635015-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 00:53:24.834987 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-fb963736-f191-4619-8e63-95dd69607b03 tempest-TestEncryptedCinderVolumes-1861635015 tempest-TestEncryptedCinderVolumes-1861635015-project] Acquiring lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 19 00:53:24.835438 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-fb963736-f191-4619-8e63-95dd69607b03 tempest-TestEncryptedCinderVolumes-1861635015 tempest-TestEncryptedCinderVolumes-1861635015-project] Acquired lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 19 00:53:24.836303 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-01c33525-e955-497a-9f7a-02a88656f5de None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 00:53:24.836824 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-01c33525-e955-497a-9f7a-02a88656f5de None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 00:53:24.837303 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-01c33525-e955-497a-9f7a-02a88656f5de None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 00:53:24.838066 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-01c33525-e955-497a-9f7a-02a88656f5de None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 00:53:24.838568 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-01c33525-e955-497a-9f7a-02a88656f5de None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 00:53:24.839098 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-01c33525-e955-497a-9f7a-02a88656f5de None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 00:53:24.848028 n-d-853704-1 nova-conductor[120657]: ERROR nova.compute.rpcapi [None req-01c33525-e955-497a-9f7a-02a88656f5de None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 19 00:53:24.848422 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-01c33525-e955-497a-9f7a-02a88656f5de None None] Releasing lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 19 00:53:24.849364 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server [None req-01c33525-e955-497a-9f7a-02a88656f5de None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 19 00:53:24.849364 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 19 00:53:24.849364 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 19 00:53:24.849364 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 19 00:53:24.849364 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 19 00:53:24.849364 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 19 00:53:24.849364 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 19 00:53:24.849364 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 19 00:53:24.849364 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 19 00:53:24.849364 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 19 00:53:24.849364 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 19 00:53:24.849364 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 19 00:53:24.849364 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 19 00:53:24.849364 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 19 00:53:24.849364 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 19 00:53:24.849364 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 19 00:53:24.849364 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 19 00:53:24.849364 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 19 00:53:24.849364 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 19 00:53:24.849364 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 19 00:53:24.849364 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 19 00:53:24.849364 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 19 00:53:24.849364 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 19 00:53:24.849364 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 19 00:53:24.849364 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 19 00:53:34.753673 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=17,nova_api:UPDATE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:53:34.754444 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=7,nova_api:DELETE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:53:34.844809 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:53:34.846575 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell1:SELECT=11,nova_cell1:INSERT=17 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:53:35.609980 n-d-853704-1 nova-conductor[120657]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 19 00:53:43.982078 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:53:44.001133 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=15,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:54:03.983942 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=3,nova_cell0:UPDATE=2 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:54:04.007858 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=3,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:54:23.988902 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=4,nova_cell0:UPDATE=2 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:54:24.008902 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=4,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:54:54.009231 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=6,nova_cell0:UPDATE=3 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:55:03.975311 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:55:34.011309 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:55:43.970602 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:56:04.017955 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=6,nova_cell0:UPDATE=3 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:56:13.233401 n-d-853704-1 nova-conductor[120657]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 19 00:56:13.973241 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=5,nova_cell0:UPDATE=3 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:56:24.024154 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=4,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:56:41.527246 n-d-853704-1 nova-conductor[120658]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 19 00:56:43.977947 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=6,nova_cell0:UPDATE=3 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:56:54.022916 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=6,nova_cell0:UPDATE=3 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:56:55.617397 n-d-853704-1 nova-conductor[120657]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 19 00:57:23.980681 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:57:24.031353 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=6,nova_cell0:UPDATE=3 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:58:03.975230 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:58:04.026408 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:58:23.980518 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=3,nova_cell0:UPDATE=2 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:58:44.033377 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:59:03.979453 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:59:14.034186 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=5,nova_cell0:UPDATE=3 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:59:26.856253 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-0edff08d-f288-44fc-9d97-0139084f1b52 tempest-TestEncryptedCinderVolumes-1861635015 tempest-TestEncryptedCinderVolumes-1861635015-project] Took 0.42 seconds to select destinations for 1 instance(s). {{(pid=120658) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 19 00:59:26.876077 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-0edff08d-f288-44fc-9d97-0139084f1b52 tempest-TestEncryptedCinderVolumes-1861635015 tempest-TestEncryptedCinderVolumes-1861635015-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 00:59:26.876517 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-0edff08d-f288-44fc-9d97-0139084f1b52 tempest-TestEncryptedCinderVolumes-1861635015 tempest-TestEncryptedCinderVolumes-1861635015-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 00:59:26.877197 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-0edff08d-f288-44fc-9d97-0139084f1b52 tempest-TestEncryptedCinderVolumes-1861635015 tempest-TestEncryptedCinderVolumes-1861635015-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 00:59:27.003775 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-0edff08d-f288-44fc-9d97-0139084f1b52 tempest-TestEncryptedCinderVolumes-1861635015 tempest-TestEncryptedCinderVolumes-1861635015-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 00:59:27.004228 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-0edff08d-f288-44fc-9d97-0139084f1b52 tempest-TestEncryptedCinderVolumes-1861635015 tempest-TestEncryptedCinderVolumes-1861635015-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 00:59:27.004656 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-0edff08d-f288-44fc-9d97-0139084f1b52 tempest-TestEncryptedCinderVolumes-1861635015 tempest-TestEncryptedCinderVolumes-1861635015-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 00:59:27.005326 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-0edff08d-f288-44fc-9d97-0139084f1b52 tempest-TestEncryptedCinderVolumes-1861635015 tempest-TestEncryptedCinderVolumes-1861635015-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 00:59:27.005743 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-0edff08d-f288-44fc-9d97-0139084f1b52 tempest-TestEncryptedCinderVolumes-1861635015 tempest-TestEncryptedCinderVolumes-1861635015-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 00:59:27.006124 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-0edff08d-f288-44fc-9d97-0139084f1b52 tempest-TestEncryptedCinderVolumes-1861635015 tempest-TestEncryptedCinderVolumes-1861635015-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 00:59:27.024642 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-0edff08d-f288-44fc-9d97-0139084f1b52 tempest-TestEncryptedCinderVolumes-1861635015 tempest-TestEncryptedCinderVolumes-1861635015-project] Getting quotas for project 9a2442d1e525437f9924a36c183502dc. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 19 00:59:27.029383 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-0edff08d-f288-44fc-9d97-0139084f1b52 tempest-TestEncryptedCinderVolumes-1861635015 tempest-TestEncryptedCinderVolumes-1861635015-project] Getting quotas for user 1588807348bf4fc2b0482241e732737e and project 9a2442d1e525437f9924a36c183502dc. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 19 00:59:27.041999 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-0edff08d-f288-44fc-9d97-0139084f1b52 tempest-TestEncryptedCinderVolumes-1861635015 tempest-TestEncryptedCinderVolumes-1861635015-project] [instance: 78a51301-07b0-450c-a77f-dfac6984859a] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120658) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 19 00:59:27.042699 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-0edff08d-f288-44fc-9d97-0139084f1b52 tempest-TestEncryptedCinderVolumes-1861635015 tempest-TestEncryptedCinderVolumes-1861635015-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 00:59:27.043163 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-0edff08d-f288-44fc-9d97-0139084f1b52 tempest-TestEncryptedCinderVolumes-1861635015 tempest-TestEncryptedCinderVolumes-1861635015-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 00:59:27.043553 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-0edff08d-f288-44fc-9d97-0139084f1b52 tempest-TestEncryptedCinderVolumes-1861635015 tempest-TestEncryptedCinderVolumes-1861635015-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 00:59:27.138381 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-0edff08d-f288-44fc-9d97-0139084f1b52 tempest-TestEncryptedCinderVolumes-1861635015 tempest-TestEncryptedCinderVolumes-1861635015-project] [instance: 78a51301-07b0-450c-a77f-dfac6984859a] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='87b0fec9-a948-41f9-a3c0-f0a6a5a8f1e7',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120658) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 19 00:59:27.139606 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-0edff08d-f288-44fc-9d97-0139084f1b52 tempest-TestEncryptedCinderVolumes-1861635015 tempest-TestEncryptedCinderVolumes-1861635015-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 00:59:27.140084 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-0edff08d-f288-44fc-9d97-0139084f1b52 tempest-TestEncryptedCinderVolumes-1861635015 tempest-TestEncryptedCinderVolumes-1861635015-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 00:59:27.140473 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-0edff08d-f288-44fc-9d97-0139084f1b52 tempest-TestEncryptedCinderVolumes-1861635015 tempest-TestEncryptedCinderVolumes-1861635015-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 00:59:27.370876 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-0edff08d-f288-44fc-9d97-0139084f1b52 tempest-TestEncryptedCinderVolumes-1861635015 tempest-TestEncryptedCinderVolumes-1861635015-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 00:59:27.371229 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-0edff08d-f288-44fc-9d97-0139084f1b52 tempest-TestEncryptedCinderVolumes-1861635015 tempest-TestEncryptedCinderVolumes-1861635015-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 00:59:27.371618 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-0edff08d-f288-44fc-9d97-0139084f1b52 tempest-TestEncryptedCinderVolumes-1861635015 tempest-TestEncryptedCinderVolumes-1861635015-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 00:59:27.372099 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-0edff08d-f288-44fc-9d97-0139084f1b52 tempest-TestEncryptedCinderVolumes-1861635015 tempest-TestEncryptedCinderVolumes-1861635015-project] Acquiring lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 19 00:59:27.372448 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-0edff08d-f288-44fc-9d97-0139084f1b52 tempest-TestEncryptedCinderVolumes-1861635015 tempest-TestEncryptedCinderVolumes-1861635015-project] Acquired lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 19 00:59:27.373194 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-462a4718-be3f-4a13-b412-cfa486520377 None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 00:59:27.373609 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-462a4718-be3f-4a13-b412-cfa486520377 None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 00:59:27.373991 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-462a4718-be3f-4a13-b412-cfa486520377 None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 00:59:27.374630 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-462a4718-be3f-4a13-b412-cfa486520377 None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 00:59:27.375034 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-462a4718-be3f-4a13-b412-cfa486520377 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 00:59:27.375577 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-462a4718-be3f-4a13-b412-cfa486520377 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 00:59:27.386300 n-d-853704-1 nova-conductor[120658]: ERROR nova.compute.rpcapi [None req-462a4718-be3f-4a13-b412-cfa486520377 None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 19 00:59:27.386685 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-462a4718-be3f-4a13-b412-cfa486520377 None None] Releasing lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 19 00:59:27.388986 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server [None req-462a4718-be3f-4a13-b412-cfa486520377 None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 19 00:59:27.388986 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 19 00:59:27.388986 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 19 00:59:27.388986 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 19 00:59:27.388986 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 19 00:59:27.388986 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 19 00:59:27.388986 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 19 00:59:27.388986 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 19 00:59:27.388986 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 19 00:59:27.388986 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 19 00:59:27.388986 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 19 00:59:27.388986 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 19 00:59:27.388986 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 19 00:59:27.388986 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 19 00:59:27.388986 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 19 00:59:27.388986 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 19 00:59:27.388986 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 19 00:59:27.388986 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 19 00:59:27.388986 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 19 00:59:27.388986 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 19 00:59:27.388986 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 19 00:59:27.388986 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 19 00:59:27.388986 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 19 00:59:27.388986 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 19 00:59:27.388986 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 19 00:59:33.241261 n-d-853704-1 nova-conductor[120657]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 19 00:59:34.041241 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=4,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:59:37.294555 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=14,nova_api:UPDATE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:59:37.295317 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=10,nova_api:DELETE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:59:37.383588 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:59:37.384750 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell1:SELECT=11,nova_cell1:INSERT=17 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 00:59:43.984681 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=13,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:00:14.046719 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:00:15.626777 n-d-853704-1 nova-conductor[120657]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 19 01:00:23.988823 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:00:34.054370 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=3,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:01:03.983096 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=7,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:01:14.042818 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=3 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:01:41.537789 n-d-853704-1 nova-conductor[120658]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: Server unexpectedly closed connection Aug 19 01:01:43.982464 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:01:54.041037 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=3 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:02:13.990507 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=5,nova_cell0:UPDATE=3 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:02:14.051409 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=3,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:02:33.993888 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=4,nova_cell0:UPDATE=2 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:02:34.053628 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=4,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:02:53.248972 n-d-853704-1 nova-conductor[120657]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 19 01:02:53.999208 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=4,nova_cell0:UPDATE=2 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:02:54.059764 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=4,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:03:33.998018 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:03:34.057792 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=3 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:03:35.635089 n-d-853704-1 nova-conductor[120657]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 19 01:04:03.999190 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=5,nova_cell0:UPDATE=3 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:04:14.060556 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:04:44.005817 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:04:54.056572 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=3 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:05:01.545211 n-d-853704-1 nova-conductor[120658]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 19 01:05:24.007296 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:05:24.056731 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=5,nova_cell0:UPDATE=3 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:06:01.475009 n-d-853704-1 nova-conductor[120657]: ERROR oslo.messaging._drivers.impl_rabbit [None req-1dd099e8-98b2-4102-8824-38aaccc88f1c tempest-TestMinimumBasicScenario-474713053 tempest-TestMinimumBasicScenario-474713053-project] [ae71ad44-1ef2-45ec-82d0-4ccfc1149604] AMQP server on 10.222.0.49:5672 is unreachable: [Errno 104] Connection reset by peer. Trying again in 1 seconds.: ConnectionResetError: [Errno 104] Connection reset by peer Aug 19 01:06:02.482007 n-d-853704-1 nova-conductor[120657]: INFO oslo.messaging._drivers.impl_rabbit [None req-1dd099e8-98b2-4102-8824-38aaccc88f1c tempest-TestMinimumBasicScenario-474713053 tempest-TestMinimumBasicScenario-474713053-project] [ae71ad44-1ef2-45ec-82d0-4ccfc1149604] Reconnected to AMQP server on 10.222.0.49:5672 via [amqp] client with port 53852. Aug 19 01:06:04.015781 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:06:04.063544 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:06:05.026243 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-1dd099e8-98b2-4102-8824-38aaccc88f1c tempest-TestMinimumBasicScenario-474713053 tempest-TestMinimumBasicScenario-474713053-project] Took 3.56 seconds to select destinations for 1 instance(s). {{(pid=120657) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 19 01:06:05.047892 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-1dd099e8-98b2-4102-8824-38aaccc88f1c tempest-TestMinimumBasicScenario-474713053 tempest-TestMinimumBasicScenario-474713053-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 01:06:05.048339 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-1dd099e8-98b2-4102-8824-38aaccc88f1c tempest-TestMinimumBasicScenario-474713053 tempest-TestMinimumBasicScenario-474713053-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 01:06:05.048735 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-1dd099e8-98b2-4102-8824-38aaccc88f1c tempest-TestMinimumBasicScenario-474713053 tempest-TestMinimumBasicScenario-474713053-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 01:06:05.295434 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-1dd099e8-98b2-4102-8824-38aaccc88f1c tempest-TestMinimumBasicScenario-474713053 tempest-TestMinimumBasicScenario-474713053-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 01:06:05.297054 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-1dd099e8-98b2-4102-8824-38aaccc88f1c tempest-TestMinimumBasicScenario-474713053 tempest-TestMinimumBasicScenario-474713053-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 01:06:05.297054 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-1dd099e8-98b2-4102-8824-38aaccc88f1c tempest-TestMinimumBasicScenario-474713053 tempest-TestMinimumBasicScenario-474713053-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 01:06:05.297054 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-1dd099e8-98b2-4102-8824-38aaccc88f1c tempest-TestMinimumBasicScenario-474713053 tempest-TestMinimumBasicScenario-474713053-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 01:06:05.297374 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-1dd099e8-98b2-4102-8824-38aaccc88f1c tempest-TestMinimumBasicScenario-474713053 tempest-TestMinimumBasicScenario-474713053-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 01:06:05.297763 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-1dd099e8-98b2-4102-8824-38aaccc88f1c tempest-TestMinimumBasicScenario-474713053 tempest-TestMinimumBasicScenario-474713053-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 01:06:05.315566 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-1dd099e8-98b2-4102-8824-38aaccc88f1c tempest-TestMinimumBasicScenario-474713053 tempest-TestMinimumBasicScenario-474713053-project] Getting quotas for project bda69fd0158443c283c43521b4fa6e11. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 19 01:06:05.321361 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-1dd099e8-98b2-4102-8824-38aaccc88f1c tempest-TestMinimumBasicScenario-474713053 tempest-TestMinimumBasicScenario-474713053-project] Getting quotas for user 64011a6860394b31853eabb3a9c89c78 and project bda69fd0158443c283c43521b4fa6e11. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 19 01:06:05.333363 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-1dd099e8-98b2-4102-8824-38aaccc88f1c tempest-TestMinimumBasicScenario-474713053 tempest-TestMinimumBasicScenario-474713053-project] [instance: 7c5d38ee-5212-4075-9457-57956b8df4d5] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120657) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 19 01:06:05.334149 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-1dd099e8-98b2-4102-8824-38aaccc88f1c tempest-TestMinimumBasicScenario-474713053 tempest-TestMinimumBasicScenario-474713053-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 01:06:05.334581 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-1dd099e8-98b2-4102-8824-38aaccc88f1c tempest-TestMinimumBasicScenario-474713053 tempest-TestMinimumBasicScenario-474713053-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 01:06:05.334989 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-1dd099e8-98b2-4102-8824-38aaccc88f1c tempest-TestMinimumBasicScenario-474713053 tempest-TestMinimumBasicScenario-474713053-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 01:06:05.399593 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-1dd099e8-98b2-4102-8824-38aaccc88f1c tempest-TestMinimumBasicScenario-474713053 tempest-TestMinimumBasicScenario-474713053-project] [instance: 7c5d38ee-5212-4075-9457-57956b8df4d5] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='3b6c5798-225e-44c7-8d91-697b34123fc2',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120657) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 19 01:06:05.400809 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-1dd099e8-98b2-4102-8824-38aaccc88f1c tempest-TestMinimumBasicScenario-474713053 tempest-TestMinimumBasicScenario-474713053-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 01:06:05.401245 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-1dd099e8-98b2-4102-8824-38aaccc88f1c tempest-TestMinimumBasicScenario-474713053 tempest-TestMinimumBasicScenario-474713053-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 01:06:05.401642 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-1dd099e8-98b2-4102-8824-38aaccc88f1c tempest-TestMinimumBasicScenario-474713053 tempest-TestMinimumBasicScenario-474713053-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 01:06:05.628963 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-1dd099e8-98b2-4102-8824-38aaccc88f1c tempest-TestMinimumBasicScenario-474713053 tempest-TestMinimumBasicScenario-474713053-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 01:06:05.629405 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-1dd099e8-98b2-4102-8824-38aaccc88f1c tempest-TestMinimumBasicScenario-474713053 tempest-TestMinimumBasicScenario-474713053-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 01:06:05.629803 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-1dd099e8-98b2-4102-8824-38aaccc88f1c tempest-TestMinimumBasicScenario-474713053 tempest-TestMinimumBasicScenario-474713053-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 01:06:05.630286 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-1dd099e8-98b2-4102-8824-38aaccc88f1c tempest-TestMinimumBasicScenario-474713053 tempest-TestMinimumBasicScenario-474713053-project] Acquiring lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 19 01:06:05.630642 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-1dd099e8-98b2-4102-8824-38aaccc88f1c tempest-TestMinimumBasicScenario-474713053 tempest-TestMinimumBasicScenario-474713053-project] Acquired lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 19 01:06:05.631371 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c14c870d-4231-4d3a-9ad5-2225dfb2d8cd None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 01:06:05.631815 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c14c870d-4231-4d3a-9ad5-2225dfb2d8cd None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 01:06:05.632199 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c14c870d-4231-4d3a-9ad5-2225dfb2d8cd None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 01:06:05.632909 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c14c870d-4231-4d3a-9ad5-2225dfb2d8cd None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 01:06:05.633318 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c14c870d-4231-4d3a-9ad5-2225dfb2d8cd None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 01:06:05.633696 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c14c870d-4231-4d3a-9ad5-2225dfb2d8cd None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 01:06:05.641759 n-d-853704-1 nova-conductor[120657]: ERROR nova.compute.rpcapi [None req-c14c870d-4231-4d3a-9ad5-2225dfb2d8cd None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 19 01:06:05.642146 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-c14c870d-4231-4d3a-9ad5-2225dfb2d8cd None None] Releasing lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 19 01:06:05.643049 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server [None req-c14c870d-4231-4d3a-9ad5-2225dfb2d8cd None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 19 01:06:05.643049 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 19 01:06:05.643049 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 19 01:06:05.643049 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 19 01:06:05.643049 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 19 01:06:05.643049 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 19 01:06:05.643049 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 19 01:06:05.643049 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 19 01:06:05.643049 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 19 01:06:05.643049 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 19 01:06:05.643049 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 19 01:06:05.643049 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 19 01:06:05.643049 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 19 01:06:05.643049 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 19 01:06:05.643049 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 19 01:06:05.643049 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 19 01:06:05.643049 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 19 01:06:05.643049 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 19 01:06:05.643049 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 19 01:06:05.643049 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 19 01:06:05.643049 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 19 01:06:05.643049 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 19 01:06:05.643049 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 19 01:06:05.643049 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 19 01:06:05.643049 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 19 01:06:15.550633 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=10,nova_api:UPDATE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:06:15.551396 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=14,nova_api:DELETE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:06:15.638577 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:06:15.639435 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell1:SELECT=14,nova_cell1:SAVEPOINT=1,nova_cell1:INSERT=15,nova_cell1:RELEASE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:06:34.016268 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=5,nova_cell0:UPDATE=3 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:06:34.067329 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=11,nova_cell0:UPDATE=3 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:06:55.643720 n-d-853704-1 nova-conductor[120657]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 19 01:07:14.014719 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:07:14.065389 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:07:54.012514 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:07:54.063105 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:08:14.015526 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=3,nova_cell0:UPDATE=2 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:08:21.552986 n-d-853704-1 nova-conductor[120658]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 19 01:08:34.067465 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:08:54.010959 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:09:04.067490 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=5,nova_cell0:UPDATE=3 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:09:34.004958 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:09:34.064820 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=6,nova_cell0:UPDATE=3 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:10:14.002399 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:10:14.063944 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:10:15.652183 n-d-853704-1 nova-conductor[120657]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 19 01:10:44.008832 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=5,nova_cell0:UPDATE=3 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:10:44.069666 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=5,nova_cell0:UPDATE=3 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:11:13.261476 n-d-853704-1 nova-conductor[120657]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 19 01:11:24.013393 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:11:24.063501 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:11:41.560796 n-d-853704-1 nova-conductor[120658]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 19 01:12:04.019981 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:12:04.067154 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=7,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:12:14.469416 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-4fea74fc-24b3-4c88-94dd-b3a37e1bcc6f tempest-TestMinimumBasicScenario-474713053 tempest-TestMinimumBasicScenario-474713053-project] Took 0.43 seconds to select destinations for 1 instance(s). {{(pid=120658) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 19 01:12:14.489502 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-4fea74fc-24b3-4c88-94dd-b3a37e1bcc6f tempest-TestMinimumBasicScenario-474713053 tempest-TestMinimumBasicScenario-474713053-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 01:12:14.492841 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-4fea74fc-24b3-4c88-94dd-b3a37e1bcc6f tempest-TestMinimumBasicScenario-474713053 tempest-TestMinimumBasicScenario-474713053-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 01:12:14.492841 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-4fea74fc-24b3-4c88-94dd-b3a37e1bcc6f tempest-TestMinimumBasicScenario-474713053 tempest-TestMinimumBasicScenario-474713053-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 01:12:14.664926 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-4fea74fc-24b3-4c88-94dd-b3a37e1bcc6f tempest-TestMinimumBasicScenario-474713053 tempest-TestMinimumBasicScenario-474713053-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 01:12:14.665298 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-4fea74fc-24b3-4c88-94dd-b3a37e1bcc6f tempest-TestMinimumBasicScenario-474713053 tempest-TestMinimumBasicScenario-474713053-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 01:12:14.665767 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-4fea74fc-24b3-4c88-94dd-b3a37e1bcc6f tempest-TestMinimumBasicScenario-474713053 tempest-TestMinimumBasicScenario-474713053-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 01:12:14.666420 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-4fea74fc-24b3-4c88-94dd-b3a37e1bcc6f tempest-TestMinimumBasicScenario-474713053 tempest-TestMinimumBasicScenario-474713053-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 01:12:14.666831 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-4fea74fc-24b3-4c88-94dd-b3a37e1bcc6f tempest-TestMinimumBasicScenario-474713053 tempest-TestMinimumBasicScenario-474713053-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 01:12:14.667213 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-4fea74fc-24b3-4c88-94dd-b3a37e1bcc6f tempest-TestMinimumBasicScenario-474713053 tempest-TestMinimumBasicScenario-474713053-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 01:12:14.686096 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-4fea74fc-24b3-4c88-94dd-b3a37e1bcc6f tempest-TestMinimumBasicScenario-474713053 tempest-TestMinimumBasicScenario-474713053-project] Getting quotas for project bda69fd0158443c283c43521b4fa6e11. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 19 01:12:14.690775 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-4fea74fc-24b3-4c88-94dd-b3a37e1bcc6f tempest-TestMinimumBasicScenario-474713053 tempest-TestMinimumBasicScenario-474713053-project] Getting quotas for user 64011a6860394b31853eabb3a9c89c78 and project bda69fd0158443c283c43521b4fa6e11. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 19 01:12:14.702315 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-4fea74fc-24b3-4c88-94dd-b3a37e1bcc6f tempest-TestMinimumBasicScenario-474713053 tempest-TestMinimumBasicScenario-474713053-project] [instance: dd9b4018-a048-4802-aaee-29c9232b1107] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120658) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 19 01:12:14.705458 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-4fea74fc-24b3-4c88-94dd-b3a37e1bcc6f tempest-TestMinimumBasicScenario-474713053 tempest-TestMinimumBasicScenario-474713053-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 01:12:14.705458 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-4fea74fc-24b3-4c88-94dd-b3a37e1bcc6f tempest-TestMinimumBasicScenario-474713053 tempest-TestMinimumBasicScenario-474713053-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 01:12:14.705458 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-4fea74fc-24b3-4c88-94dd-b3a37e1bcc6f tempest-TestMinimumBasicScenario-474713053 tempest-TestMinimumBasicScenario-474713053-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 01:12:14.792718 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-4fea74fc-24b3-4c88-94dd-b3a37e1bcc6f tempest-TestMinimumBasicScenario-474713053 tempest-TestMinimumBasicScenario-474713053-project] [instance: dd9b4018-a048-4802-aaee-29c9232b1107] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='932d6337-144e-48c3-a280-9eceda12a079',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120658) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 19 01:12:14.794574 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-4fea74fc-24b3-4c88-94dd-b3a37e1bcc6f tempest-TestMinimumBasicScenario-474713053 tempest-TestMinimumBasicScenario-474713053-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 01:12:14.795008 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-4fea74fc-24b3-4c88-94dd-b3a37e1bcc6f tempest-TestMinimumBasicScenario-474713053 tempest-TestMinimumBasicScenario-474713053-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 01:12:14.795396 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-4fea74fc-24b3-4c88-94dd-b3a37e1bcc6f tempest-TestMinimumBasicScenario-474713053 tempest-TestMinimumBasicScenario-474713053-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 01:12:15.010627 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-4fea74fc-24b3-4c88-94dd-b3a37e1bcc6f tempest-TestMinimumBasicScenario-474713053 tempest-TestMinimumBasicScenario-474713053-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 01:12:15.010960 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-4fea74fc-24b3-4c88-94dd-b3a37e1bcc6f tempest-TestMinimumBasicScenario-474713053 tempest-TestMinimumBasicScenario-474713053-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 01:12:15.011347 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-4fea74fc-24b3-4c88-94dd-b3a37e1bcc6f tempest-TestMinimumBasicScenario-474713053 tempest-TestMinimumBasicScenario-474713053-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 01:12:15.011821 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-4fea74fc-24b3-4c88-94dd-b3a37e1bcc6f tempest-TestMinimumBasicScenario-474713053 tempest-TestMinimumBasicScenario-474713053-project] Acquiring lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 19 01:12:15.012199 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-4fea74fc-24b3-4c88-94dd-b3a37e1bcc6f tempest-TestMinimumBasicScenario-474713053 tempest-TestMinimumBasicScenario-474713053-project] Acquired lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 19 01:12:15.012917 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-9a1319ed-90eb-4a4f-8605-e30e69048611 None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 01:12:15.013327 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-9a1319ed-90eb-4a4f-8605-e30e69048611 None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 01:12:15.013699 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-9a1319ed-90eb-4a4f-8605-e30e69048611 None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 01:12:15.014331 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-9a1319ed-90eb-4a4f-8605-e30e69048611 None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 01:12:15.014731 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-9a1319ed-90eb-4a4f-8605-e30e69048611 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 01:12:15.015103 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-9a1319ed-90eb-4a4f-8605-e30e69048611 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 01:12:15.023033 n-d-853704-1 nova-conductor[120658]: ERROR nova.compute.rpcapi [None req-9a1319ed-90eb-4a4f-8605-e30e69048611 None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 19 01:12:15.023419 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-9a1319ed-90eb-4a4f-8605-e30e69048611 None None] Releasing lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 19 01:12:15.024366 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server [None req-9a1319ed-90eb-4a4f-8605-e30e69048611 None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 19 01:12:15.024366 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 19 01:12:15.024366 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 19 01:12:15.024366 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 19 01:12:15.024366 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 19 01:12:15.024366 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 19 01:12:15.024366 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 19 01:12:15.024366 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 19 01:12:15.024366 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 19 01:12:15.024366 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 19 01:12:15.024366 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 19 01:12:15.024366 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 19 01:12:15.024366 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 19 01:12:15.024366 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 19 01:12:15.024366 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 19 01:12:15.024366 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 19 01:12:15.024366 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 19 01:12:15.024366 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 19 01:12:15.024366 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 19 01:12:15.024366 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 19 01:12:15.024366 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 19 01:12:15.024366 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 19 01:12:15.024366 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 19 01:12:15.024366 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 19 01:12:15.024366 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 19 01:12:24.948671 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=14 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:12:24.949481 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=10,nova_api:UPDATE=1,nova_api:DELETE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:12:25.020142 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:12:25.021305 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell1:SELECT=14,nova_cell1:INSERT=14 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:12:44.018037 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=14,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:12:44.069726 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:13:24.026016 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:13:24.078584 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:13:35.660036 n-d-853704-1 nova-conductor[120657]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 19 01:14:04.033191 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=7,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:14:04.083595 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=7,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:14:33.273714 n-d-853704-1 nova-conductor[120657]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 19 01:14:34.042674 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=6,nova_cell0:UPDATE=3 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:14:34.084177 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=6,nova_cell0:UPDATE=3 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:14:54.051771 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=4,nova_cell0:UPDATE=2 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:15:01.568573 n-d-853704-1 nova-conductor[120658]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 19 01:15:14.084660 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:15:34.061840 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:15:34.090854 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=3,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:15:54.072357 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=4,nova_cell0:UPDATE=2 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:15:54.103173 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=4,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:16:34.071298 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:16:34.102066 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:16:55.667857 n-d-853704-1 nova-conductor[120657]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 19 01:17:14.078090 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:17:14.108220 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:17:34.085363 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=3,nova_cell0:UPDATE=2 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:17:53.281417 n-d-853704-1 nova-conductor[120657]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 19 01:17:54.091505 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=4,nova_cell0:UPDATE=2 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:17:54.121664 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:18:21.576595 n-d-853704-1 nova-conductor[120658]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 19 01:18:34.095728 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:18:34.121366 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:18:38.027451 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-fea85d98-cfd9-4db5-be9a-ab52a9f4aa1b tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] Took 0.72 seconds to select destinations for 1 instance(s). {{(pid=120657) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 19 01:18:38.048596 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-fea85d98-cfd9-4db5-be9a-ab52a9f4aa1b tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 01:18:38.049039 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-fea85d98-cfd9-4db5-be9a-ab52a9f4aa1b tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 01:18:38.049430 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-fea85d98-cfd9-4db5-be9a-ab52a9f4aa1b tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 01:18:38.254746 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-fea85d98-cfd9-4db5-be9a-ab52a9f4aa1b tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 01:18:38.255240 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-fea85d98-cfd9-4db5-be9a-ab52a9f4aa1b tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 01:18:38.255635 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-fea85d98-cfd9-4db5-be9a-ab52a9f4aa1b tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 01:18:38.256636 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-fea85d98-cfd9-4db5-be9a-ab52a9f4aa1b tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 01:18:38.256747 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-fea85d98-cfd9-4db5-be9a-ab52a9f4aa1b tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 01:18:38.257103 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-fea85d98-cfd9-4db5-be9a-ab52a9f4aa1b tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 01:18:38.283144 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-fea85d98-cfd9-4db5-be9a-ab52a9f4aa1b tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] Getting quotas for project b4b2d944c77a422b8e14fc1c415e47d4. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 19 01:18:38.290524 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-fea85d98-cfd9-4db5-be9a-ab52a9f4aa1b tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] Getting quotas for user 6a11720d1ca64c449c36b52c0934df46 and project b4b2d944c77a422b8e14fc1c415e47d4. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 19 01:18:38.302806 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-fea85d98-cfd9-4db5-be9a-ab52a9f4aa1b tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] [instance: 88baf564-ba05-4b24-8795-dc49a9364cbe] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120657) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 19 01:18:38.303584 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-fea85d98-cfd9-4db5-be9a-ab52a9f4aa1b tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 01:18:38.304048 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-fea85d98-cfd9-4db5-be9a-ab52a9f4aa1b tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 01:18:38.304435 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-fea85d98-cfd9-4db5-be9a-ab52a9f4aa1b tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 01:18:38.358977 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-fea85d98-cfd9-4db5-be9a-ab52a9f4aa1b tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] [instance: 88baf564-ba05-4b24-8795-dc49a9364cbe] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='87b0fec9-a948-41f9-a3c0-f0a6a5a8f1e7',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120657) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 19 01:18:38.358977 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-fea85d98-cfd9-4db5-be9a-ab52a9f4aa1b tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 01:18:38.358977 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-fea85d98-cfd9-4db5-be9a-ab52a9f4aa1b tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 01:18:38.358977 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-fea85d98-cfd9-4db5-be9a-ab52a9f4aa1b tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 01:18:38.608235 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-fea85d98-cfd9-4db5-be9a-ab52a9f4aa1b tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 01:18:38.609027 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-fea85d98-cfd9-4db5-be9a-ab52a9f4aa1b tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.002s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 01:18:38.609647 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-fea85d98-cfd9-4db5-be9a-ab52a9f4aa1b tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 01:18:38.610676 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-fea85d98-cfd9-4db5-be9a-ab52a9f4aa1b tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] Acquiring lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 19 01:18:38.611092 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-fea85d98-cfd9-4db5-be9a-ab52a9f4aa1b tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] Acquired lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 19 01:18:38.611902 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-143999a7-5425-480e-b660-25cd9a71e671 None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 01:18:38.612560 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-143999a7-5425-480e-b660-25cd9a71e671 None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 01:18:38.627934 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-143999a7-5425-480e-b660-25cd9a71e671 None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 01:18:38.627934 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-143999a7-5425-480e-b660-25cd9a71e671 None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 01:18:38.627934 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-143999a7-5425-480e-b660-25cd9a71e671 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 01:18:38.627934 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-143999a7-5425-480e-b660-25cd9a71e671 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 01:18:38.627934 n-d-853704-1 nova-conductor[120657]: ERROR nova.compute.rpcapi [None req-143999a7-5425-480e-b660-25cd9a71e671 None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 19 01:18:38.627934 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-143999a7-5425-480e-b660-25cd9a71e671 None None] Releasing lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 19 01:18:38.628981 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server [None req-143999a7-5425-480e-b660-25cd9a71e671 None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 19 01:18:38.628981 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 19 01:18:38.628981 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 19 01:18:38.628981 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 19 01:18:38.628981 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 19 01:18:38.628981 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 19 01:18:38.628981 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 19 01:18:38.628981 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 19 01:18:38.628981 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 19 01:18:38.628981 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 19 01:18:38.628981 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 19 01:18:38.628981 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 19 01:18:38.628981 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 19 01:18:38.628981 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 19 01:18:38.628981 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 19 01:18:38.628981 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 19 01:18:38.628981 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 19 01:18:38.628981 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 19 01:18:38.628981 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 19 01:18:38.628981 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 19 01:18:38.628981 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 19 01:18:38.628981 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 19 01:18:38.628981 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 19 01:18:38.628981 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 19 01:18:38.628981 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 19 01:18:48.519755 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=13,nova_api:UPDATE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:18:48.521623 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=11,nova_api:DELETE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:18:48.623957 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:18:48.624594 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell1:SELECT=11,nova_cell1:SAVEPOINT=1,nova_cell1:INSERT=18,nova_cell1:RELEASE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:19:14.094716 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:19:14.126792 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=14,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:19:54.091688 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:19:54.123442 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:20:15.675767 n-d-853704-1 nova-conductor[120657]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 19 01:20:34.100715 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:20:34.132939 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:21:13.292001 n-d-853704-1 nova-conductor[120657]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 19 01:21:14.105479 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:21:14.135817 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:21:41.584706 n-d-853704-1 nova-conductor[120658]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 19 01:21:54.108957 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:21:54.140816 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:22:14.144150 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=3,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:22:34.109733 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:22:44.150153 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=6,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:22:54.112900 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=3,nova_cell0:UPDATE=2 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:23:04.158755 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=4,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:23:34.121981 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:23:35.685664 n-d-853704-1 nova-conductor[120657]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 19 01:23:44.165107 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:23:54.128452 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=3,nova_cell0:UPDATE=2 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:24:24.170180 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:24:33.300808 n-d-853704-1 nova-conductor[120657]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 19 01:24:34.132577 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:24:44.180116 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=3,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:24:51.432290 n-d-853704-1 nova-conductor[120658]: ERROR oslo.messaging._drivers.impl_rabbit [None req-f6373bc7-38bd-4bc7-9398-a5dc8bc5f13e tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] [fe4e6e0b-b1ce-4fa1-b3d0-5231d7497da8] AMQP server on 10.222.0.49:5672 is unreachable: [Errno 104] Connection reset by peer. Trying again in 1 seconds.: ConnectionResetError: [Errno 104] Connection reset by peer Aug 19 01:24:52.439075 n-d-853704-1 nova-conductor[120658]: INFO oslo.messaging._drivers.impl_rabbit [None req-f6373bc7-38bd-4bc7-9398-a5dc8bc5f13e tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] [fe4e6e0b-b1ce-4fa1-b3d0-5231d7497da8] Reconnected to AMQP server on 10.222.0.49:5672 via [amqp] client with port 56378. Aug 19 01:24:54.914884 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-f6373bc7-38bd-4bc7-9398-a5dc8bc5f13e tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] Took 3.49 seconds to select destinations for 1 instance(s). {{(pid=120658) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 19 01:24:54.934440 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-f6373bc7-38bd-4bc7-9398-a5dc8bc5f13e tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 01:24:54.936276 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-f6373bc7-38bd-4bc7-9398-a5dc8bc5f13e tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 01:24:54.936276 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-f6373bc7-38bd-4bc7-9398-a5dc8bc5f13e tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 01:24:55.069458 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-f6373bc7-38bd-4bc7-9398-a5dc8bc5f13e tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 01:24:55.072473 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-f6373bc7-38bd-4bc7-9398-a5dc8bc5f13e tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 01:24:55.072473 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-f6373bc7-38bd-4bc7-9398-a5dc8bc5f13e tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 01:24:55.072473 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-f6373bc7-38bd-4bc7-9398-a5dc8bc5f13e tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 01:24:55.072813 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-f6373bc7-38bd-4bc7-9398-a5dc8bc5f13e tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 01:24:55.073044 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-f6373bc7-38bd-4bc7-9398-a5dc8bc5f13e tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 01:24:55.088391 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-f6373bc7-38bd-4bc7-9398-a5dc8bc5f13e tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] Getting quotas for project b4b2d944c77a422b8e14fc1c415e47d4. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 19 01:24:55.092958 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-f6373bc7-38bd-4bc7-9398-a5dc8bc5f13e tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] Getting quotas for user 6a11720d1ca64c449c36b52c0934df46 and project b4b2d944c77a422b8e14fc1c415e47d4. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 19 01:24:55.104014 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-f6373bc7-38bd-4bc7-9398-a5dc8bc5f13e tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] [instance: d345204c-482d-4799-b366-c352462fc0bd] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120658) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 19 01:24:55.104790 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-f6373bc7-38bd-4bc7-9398-a5dc8bc5f13e tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 01:24:55.105232 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-f6373bc7-38bd-4bc7-9398-a5dc8bc5f13e tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 01:24:55.105621 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-f6373bc7-38bd-4bc7-9398-a5dc8bc5f13e tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 01:24:55.178891 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-f6373bc7-38bd-4bc7-9398-a5dc8bc5f13e tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] [instance: d345204c-482d-4799-b366-c352462fc0bd] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='87b0fec9-a948-41f9-a3c0-f0a6a5a8f1e7',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120658) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 19 01:24:55.180754 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-f6373bc7-38bd-4bc7-9398-a5dc8bc5f13e tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 01:24:55.181191 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-f6373bc7-38bd-4bc7-9398-a5dc8bc5f13e tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 01:24:55.181587 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-f6373bc7-38bd-4bc7-9398-a5dc8bc5f13e tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 01:24:55.436734 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-f6373bc7-38bd-4bc7-9398-a5dc8bc5f13e tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 01:24:55.437108 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-f6373bc7-38bd-4bc7-9398-a5dc8bc5f13e tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 01:24:55.437513 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-f6373bc7-38bd-4bc7-9398-a5dc8bc5f13e tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 01:24:55.437966 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-f6373bc7-38bd-4bc7-9398-a5dc8bc5f13e tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] Acquiring lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 19 01:24:55.438316 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-f6373bc7-38bd-4bc7-9398-a5dc8bc5f13e tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] Acquired lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 19 01:24:55.439156 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-5166eb3d-c182-4c8b-8764-c625ad2f091f None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 01:24:55.439585 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-5166eb3d-c182-4c8b-8764-c625ad2f091f None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 01:24:55.440105 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-5166eb3d-c182-4c8b-8764-c625ad2f091f None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 01:24:55.440677 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-5166eb3d-c182-4c8b-8764-c625ad2f091f None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 01:24:55.441084 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-5166eb3d-c182-4c8b-8764-c625ad2f091f None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 01:24:55.441463 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-5166eb3d-c182-4c8b-8764-c625ad2f091f None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 01:24:55.451723 n-d-853704-1 nova-conductor[120658]: ERROR nova.compute.rpcapi [None req-5166eb3d-c182-4c8b-8764-c625ad2f091f None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 19 01:24:55.452107 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-5166eb3d-c182-4c8b-8764-c625ad2f091f None None] Releasing lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 19 01:24:55.453010 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server [None req-5166eb3d-c182-4c8b-8764-c625ad2f091f None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 19 01:24:55.453010 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 19 01:24:55.453010 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 19 01:24:55.453010 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 19 01:24:55.453010 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 19 01:24:55.453010 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 19 01:24:55.453010 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 19 01:24:55.453010 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 19 01:24:55.453010 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 19 01:24:55.453010 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 19 01:24:55.453010 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 19 01:24:55.453010 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 19 01:24:55.453010 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 19 01:24:55.453010 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 19 01:24:55.453010 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 19 01:24:55.453010 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 19 01:24:55.453010 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 19 01:24:55.453010 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 19 01:24:55.453010 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 19 01:24:55.453010 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 19 01:24:55.453010 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 19 01:24:55.453010 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 19 01:24:55.453010 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 19 01:24:55.453010 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 19 01:24:55.453010 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 19 01:25:05.346711 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=13,nova_api:UPDATE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:25:05.347526 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=11,nova_api:DELETE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:25:05.449026 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:25:05.449619 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell1:SELECT=11,nova_cell1:INSERT=17 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:25:14.144136 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=14,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:25:14.185242 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=6,nova_cell0:UPDATE=3 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:25:34.196151 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=4,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:25:44.151129 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=6,nova_cell0:UPDATE=3 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:26:14.204777 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:26:24.152373 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:26:34.207047 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=3,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:26:55.694358 n-d-853704-1 nova-conductor[120657]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 19 01:27:04.154490 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:27:14.200530 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:27:34.164082 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=5,nova_cell0:UPDATE=3 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:27:34.205258 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=3,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:27:53.310048 n-d-853704-1 nova-conductor[120657]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 19 01:27:54.212507 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=4,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:28:14.174152 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:28:14.215478 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=4,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:28:54.170856 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:28:54.212426 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:29:14.174622 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=3,nova_cell0:UPDATE=2 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:29:34.217295 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:29:54.183436 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:30:01.598191 n-d-853704-1 nova-conductor[120658]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 19 01:30:14.225006 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:30:15.702395 n-d-853704-1 nova-conductor[120657]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 19 01:30:34.184685 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:30:54.232164 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=7,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:31:04.207473 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=5,nova_cell0:UPDATE=3 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:31:06.251859 n-d-853704-1 nova-conductor[120657]: ERROR oslo.messaging._drivers.impl_rabbit [None req-25546108-e158-4142-9794-b81c5c4e1f09 tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] [ae71ad44-1ef2-45ec-82d0-4ccfc1149604] AMQP server on 10.222.0.49:5672 is unreachable: [Errno 104] Connection reset by peer. Trying again in 1 seconds.: ConnectionResetError: [Errno 104] Connection reset by peer Aug 19 01:31:07.258927 n-d-853704-1 nova-conductor[120657]: INFO oslo.messaging._drivers.impl_rabbit [None req-25546108-e158-4142-9794-b81c5c4e1f09 tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] [ae71ad44-1ef2-45ec-82d0-4ccfc1149604] Reconnected to AMQP server on 10.222.0.49:5672 via [amqp] client with port 57174. Aug 19 01:31:09.728288 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-25546108-e158-4142-9794-b81c5c4e1f09 tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] Took 3.48 seconds to select destinations for 1 instance(s). {{(pid=120657) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 19 01:31:09.749215 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-25546108-e158-4142-9794-b81c5c4e1f09 tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 01:31:09.749659 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-25546108-e158-4142-9794-b81c5c4e1f09 tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 01:31:09.750054 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-25546108-e158-4142-9794-b81c5c4e1f09 tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 01:31:09.867705 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-25546108-e158-4142-9794-b81c5c4e1f09 tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 01:31:09.868078 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-25546108-e158-4142-9794-b81c5c4e1f09 tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 01:31:09.868474 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-25546108-e158-4142-9794-b81c5c4e1f09 tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 01:31:09.869136 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-25546108-e158-4142-9794-b81c5c4e1f09 tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 01:31:09.869548 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-25546108-e158-4142-9794-b81c5c4e1f09 tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 01:31:09.869932 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-25546108-e158-4142-9794-b81c5c4e1f09 tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 01:31:09.885808 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-25546108-e158-4142-9794-b81c5c4e1f09 tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] Getting quotas for project b4b2d944c77a422b8e14fc1c415e47d4. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 19 01:31:09.890248 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-25546108-e158-4142-9794-b81c5c4e1f09 tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] Getting quotas for user 6a11720d1ca64c449c36b52c0934df46 and project b4b2d944c77a422b8e14fc1c415e47d4. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 19 01:31:09.901987 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-25546108-e158-4142-9794-b81c5c4e1f09 tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] [instance: f999bb7f-3461-434f-9091-c7491dae8b6d] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120657) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 19 01:31:09.902748 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-25546108-e158-4142-9794-b81c5c4e1f09 tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 01:31:09.903183 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-25546108-e158-4142-9794-b81c5c4e1f09 tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 01:31:09.903577 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-25546108-e158-4142-9794-b81c5c4e1f09 tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 01:31:09.954432 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-25546108-e158-4142-9794-b81c5c4e1f09 tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] [instance: f999bb7f-3461-434f-9091-c7491dae8b6d] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='87b0fec9-a948-41f9-a3c0-f0a6a5a8f1e7',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120657) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 19 01:31:09.956420 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-25546108-e158-4142-9794-b81c5c4e1f09 tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 01:31:09.956859 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-25546108-e158-4142-9794-b81c5c4e1f09 tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 01:31:09.957256 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-25546108-e158-4142-9794-b81c5c4e1f09 tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 01:31:10.243575 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-25546108-e158-4142-9794-b81c5c4e1f09 tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 01:31:10.244031 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-25546108-e158-4142-9794-b81c5c4e1f09 tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 01:31:10.244437 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-25546108-e158-4142-9794-b81c5c4e1f09 tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 01:31:10.244893 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-25546108-e158-4142-9794-b81c5c4e1f09 tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] Acquiring lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 19 01:31:10.245251 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-25546108-e158-4142-9794-b81c5c4e1f09 tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] Acquired lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 19 01:31:10.245989 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-8b4382a0-de18-4e90-84fe-53cce38205e9 None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 01:31:10.246432 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-8b4382a0-de18-4e90-84fe-53cce38205e9 None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 01:31:10.246871 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-8b4382a0-de18-4e90-84fe-53cce38205e9 None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 01:31:10.247470 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-8b4382a0-de18-4e90-84fe-53cce38205e9 None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 01:31:10.248030 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-8b4382a0-de18-4e90-84fe-53cce38205e9 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 01:31:10.248420 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-8b4382a0-de18-4e90-84fe-53cce38205e9 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 01:31:10.257370 n-d-853704-1 nova-conductor[120657]: ERROR nova.compute.rpcapi [None req-8b4382a0-de18-4e90-84fe-53cce38205e9 None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 19 01:31:10.257767 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-8b4382a0-de18-4e90-84fe-53cce38205e9 None None] Releasing lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 19 01:31:10.258668 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server [None req-8b4382a0-de18-4e90-84fe-53cce38205e9 None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 19 01:31:10.258668 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 19 01:31:10.258668 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 19 01:31:10.258668 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 19 01:31:10.258668 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 19 01:31:10.258668 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 19 01:31:10.258668 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 19 01:31:10.258668 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 19 01:31:10.258668 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 19 01:31:10.258668 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 19 01:31:10.258668 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 19 01:31:10.258668 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 19 01:31:10.258668 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 19 01:31:10.258668 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 19 01:31:10.258668 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 19 01:31:10.258668 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 19 01:31:10.258668 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 19 01:31:10.258668 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 19 01:31:10.258668 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 19 01:31:10.258668 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 19 01:31:10.258668 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 19 01:31:10.258668 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 19 01:31:10.258668 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 19 01:31:10.258668 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 19 01:31:10.258668 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 19 01:31:20.151458 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=13,nova_api:UPDATE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:31:20.152315 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=11,nova_api:DELETE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:31:20.254490 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:31:20.255112 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell1:SELECT=11,nova_cell1:INSERT=17 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:31:24.199321 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=4,nova_cell0:UPDATE=2 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:31:34.243972 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=14,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:31:54.208212 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=6,nova_cell0:UPDATE=3 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:31:54.248007 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=4,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:32:14.213807 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=4,nova_cell0:UPDATE=2 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:32:14.255799 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=4,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:32:34.263129 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=4,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:32:54.219337 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:33:14.277982 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:33:21.606405 n-d-853704-1 nova-conductor[120658]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 19 01:33:34.229143 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=7,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:33:34.277835 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=4,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:33:35.713481 n-d-853704-1 nova-conductor[120657]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 19 01:34:14.227999 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:34:14.277603 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:34:34.233433 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=3,nova_cell0:UPDATE=2 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:34:34.282871 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=3,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:34:54.288988 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=4,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:35:04.243346 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=6,nova_cell0:UPDATE=3 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:35:14.297380 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=4,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:35:44.242671 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:35:54.293144 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:36:13.322515 n-d-853704-1 nova-conductor[120657]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 19 01:36:24.241652 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:36:34.295275 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=7,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:36:41.614318 n-d-853704-1 nova-conductor[120658]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 19 01:36:55.723217 n-d-853704-1 nova-conductor[120657]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 19 01:37:04.246656 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:37:14.295326 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:37:20.144844 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-e9418a2f-8ce4-4866-a6b2-6c3fb3fde736 tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] Took 0.71 seconds to select destinations for 1 instance(s). {{(pid=120658) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 19 01:37:20.167840 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-e9418a2f-8ce4-4866-a6b2-6c3fb3fde736 tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 01:37:20.168374 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-e9418a2f-8ce4-4866-a6b2-6c3fb3fde736 tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 01:37:20.169052 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-e9418a2f-8ce4-4866-a6b2-6c3fb3fde736 tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 01:37:20.327763 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-e9418a2f-8ce4-4866-a6b2-6c3fb3fde736 tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 01:37:20.328213 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-e9418a2f-8ce4-4866-a6b2-6c3fb3fde736 tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 01:37:20.328610 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-e9418a2f-8ce4-4866-a6b2-6c3fb3fde736 tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 01:37:20.329287 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-e9418a2f-8ce4-4866-a6b2-6c3fb3fde736 tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 01:37:20.329689 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-e9418a2f-8ce4-4866-a6b2-6c3fb3fde736 tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 01:37:20.330072 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-e9418a2f-8ce4-4866-a6b2-6c3fb3fde736 tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 01:37:20.348018 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-e9418a2f-8ce4-4866-a6b2-6c3fb3fde736 tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] Getting quotas for project b4b2d944c77a422b8e14fc1c415e47d4. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 19 01:37:20.352844 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-e9418a2f-8ce4-4866-a6b2-6c3fb3fde736 tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] Getting quotas for user 6a11720d1ca64c449c36b52c0934df46 and project b4b2d944c77a422b8e14fc1c415e47d4. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 19 01:37:20.364919 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-e9418a2f-8ce4-4866-a6b2-6c3fb3fde736 tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] [instance: 0fd05d8b-d422-4744-9511-9ff36e672cc0] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120658) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 19 01:37:20.365703 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-e9418a2f-8ce4-4866-a6b2-6c3fb3fde736 tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 01:37:20.366139 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-e9418a2f-8ce4-4866-a6b2-6c3fb3fde736 tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 01:37:20.366530 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-e9418a2f-8ce4-4866-a6b2-6c3fb3fde736 tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 01:37:20.438832 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-e9418a2f-8ce4-4866-a6b2-6c3fb3fde736 tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] [instance: 0fd05d8b-d422-4744-9511-9ff36e672cc0] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='87b0fec9-a948-41f9-a3c0-f0a6a5a8f1e7',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120658) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 19 01:37:20.441868 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-e9418a2f-8ce4-4866-a6b2-6c3fb3fde736 tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 01:37:20.441868 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-e9418a2f-8ce4-4866-a6b2-6c3fb3fde736 tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 01:37:20.442107 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-e9418a2f-8ce4-4866-a6b2-6c3fb3fde736 tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 01:37:20.719285 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-e9418a2f-8ce4-4866-a6b2-6c3fb3fde736 tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 01:37:20.719789 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-e9418a2f-8ce4-4866-a6b2-6c3fb3fde736 tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 01:37:20.720351 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-e9418a2f-8ce4-4866-a6b2-6c3fb3fde736 tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 01:37:20.720732 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-e9418a2f-8ce4-4866-a6b2-6c3fb3fde736 tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] Acquiring lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 19 01:37:20.720978 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-e9418a2f-8ce4-4866-a6b2-6c3fb3fde736 tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] Acquired lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 19 01:37:20.722073 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-0f786a02-a379-4926-b94d-16c386ec1f97 None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 01:37:20.722545 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-0f786a02-a379-4926-b94d-16c386ec1f97 None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 01:37:20.722978 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-0f786a02-a379-4926-b94d-16c386ec1f97 None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 01:37:20.723782 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-0f786a02-a379-4926-b94d-16c386ec1f97 None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 01:37:20.724332 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-0f786a02-a379-4926-b94d-16c386ec1f97 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 01:37:20.724733 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-0f786a02-a379-4926-b94d-16c386ec1f97 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 01:37:20.733138 n-d-853704-1 nova-conductor[120658]: ERROR nova.compute.rpcapi [None req-0f786a02-a379-4926-b94d-16c386ec1f97 None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 19 01:37:20.738625 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-0f786a02-a379-4926-b94d-16c386ec1f97 None None] Releasing lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 19 01:37:20.738625 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server [None req-0f786a02-a379-4926-b94d-16c386ec1f97 None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 19 01:37:20.738625 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 19 01:37:20.738625 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 19 01:37:20.738625 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 19 01:37:20.738625 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 19 01:37:20.738625 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 19 01:37:20.738625 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 19 01:37:20.738625 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 19 01:37:20.738625 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 19 01:37:20.738625 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 19 01:37:20.738625 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 19 01:37:20.738625 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 19 01:37:20.738625 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 19 01:37:20.738625 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 19 01:37:20.738625 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 19 01:37:20.738625 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 19 01:37:20.738625 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 19 01:37:20.738625 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 19 01:37:20.738625 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 19 01:37:20.738625 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 19 01:37:20.738625 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 19 01:37:20.738625 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 19 01:37:20.738625 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 19 01:37:20.738625 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 19 01:37:20.738625 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 19 01:37:30.524489 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=11,nova_api:UPDATE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:37:30.613454 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=13,nova_api:DELETE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:37:30.731997 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:37:30.732677 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell1:SELECT=11,nova_cell1:INSERT=17 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:37:34.248213 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=11,nova_cell0:UPDATE=3 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:37:34.294643 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=3,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:38:04.245234 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=6,nova_cell0:UPDATE=3 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:38:04.294758 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=6,nova_cell0:UPDATE=3 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:38:44.242867 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:38:44.294056 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:39:24.236877 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:39:24.296421 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:39:33.330486 n-d-853704-1 nova-conductor[120657]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 19 01:39:44.241872 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=3,nova_cell0:UPDATE=2 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:40:01.622214 n-d-853704-1 nova-conductor[120658]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 19 01:40:04.251814 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=4,nova_cell0:UPDATE=2 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:40:04.292190 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:40:15.731264 n-d-853704-1 nova-conductor[120657]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 19 01:40:34.263762 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=6,nova_cell0:UPDATE=3 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:40:44.293446 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:40:54.266988 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=4,nova_cell0:UPDATE=2 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:41:04.303748 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=3,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:41:24.308350 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=4,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:41:34.278558 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:41:44.312236 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=4,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:42:14.278709 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:42:24.311635 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:42:53.339878 n-d-853704-1 nova-conductor[120657]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 19 01:42:54.281800 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=7,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:43:04.308517 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:43:21.629996 n-d-853704-1 nova-conductor[120658]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 19 01:43:24.317714 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=3,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:43:33.960988 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-b4b34614-9e44-4c00-bdef-64961f124960 tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] Took 0.65 seconds to select destinations for 1 instance(s). {{(pid=120657) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 19 01:43:33.993609 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-b4b34614-9e44-4c00-bdef-64961f124960 tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 01:43:33.994062 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-b4b34614-9e44-4c00-bdef-64961f124960 tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 01:43:33.994456 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-b4b34614-9e44-4c00-bdef-64961f124960 tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 01:43:34.117788 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-b4b34614-9e44-4c00-bdef-64961f124960 tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 01:43:34.118235 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-b4b34614-9e44-4c00-bdef-64961f124960 tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 01:43:34.118634 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-b4b34614-9e44-4c00-bdef-64961f124960 tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 01:43:34.119303 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-b4b34614-9e44-4c00-bdef-64961f124960 tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 01:43:34.119740 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-b4b34614-9e44-4c00-bdef-64961f124960 tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 01:43:34.120131 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-b4b34614-9e44-4c00-bdef-64961f124960 tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 01:43:34.141669 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-b4b34614-9e44-4c00-bdef-64961f124960 tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] Getting quotas for project b4b2d944c77a422b8e14fc1c415e47d4. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 19 01:43:34.146784 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-b4b34614-9e44-4c00-bdef-64961f124960 tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] Getting quotas for user 6a11720d1ca64c449c36b52c0934df46 and project b4b2d944c77a422b8e14fc1c415e47d4. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 19 01:43:34.158463 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-b4b34614-9e44-4c00-bdef-64961f124960 tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] [instance: 6ebf44c6-bf14-4302-9659-c5dfbeb31ded] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120657) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 19 01:43:34.159333 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-b4b34614-9e44-4c00-bdef-64961f124960 tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 01:43:34.159800 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-b4b34614-9e44-4c00-bdef-64961f124960 tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 01:43:34.160196 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-b4b34614-9e44-4c00-bdef-64961f124960 tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 01:43:34.215196 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-b4b34614-9e44-4c00-bdef-64961f124960 tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] [instance: 6ebf44c6-bf14-4302-9659-c5dfbeb31ded] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='87b0fec9-a948-41f9-a3c0-f0a6a5a8f1e7',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120657) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 19 01:43:34.216553 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-b4b34614-9e44-4c00-bdef-64961f124960 tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 01:43:34.216998 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-b4b34614-9e44-4c00-bdef-64961f124960 tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 01:43:34.217493 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-b4b34614-9e44-4c00-bdef-64961f124960 tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 01:43:34.284379 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:43:34.456773 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-b4b34614-9e44-4c00-bdef-64961f124960 tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 01:43:34.457497 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-b4b34614-9e44-4c00-bdef-64961f124960 tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 01:43:34.457896 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-b4b34614-9e44-4c00-bdef-64961f124960 tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 01:43:34.458461 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-b4b34614-9e44-4c00-bdef-64961f124960 tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] Acquiring lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 19 01:43:34.459018 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-b4b34614-9e44-4c00-bdef-64961f124960 tempest-TestNetworkAdvancedServerOps-1941596817 tempest-TestNetworkAdvancedServerOps-1941596817-project] Acquired lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 19 01:43:34.460284 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-b85036e1-daf9-4dd5-af20-c56b4f72d818 None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 01:43:34.460729 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-b85036e1-daf9-4dd5-af20-c56b4f72d818 None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 01:43:34.470338 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-b85036e1-daf9-4dd5-af20-c56b4f72d818 None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 01:43:34.470338 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-b85036e1-daf9-4dd5-af20-c56b4f72d818 None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 01:43:34.470338 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-b85036e1-daf9-4dd5-af20-c56b4f72d818 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 01:43:34.470338 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-b85036e1-daf9-4dd5-af20-c56b4f72d818 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 01:43:34.473815 n-d-853704-1 nova-conductor[120657]: ERROR nova.compute.rpcapi [None req-b85036e1-daf9-4dd5-af20-c56b4f72d818 None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 19 01:43:34.473943 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-b85036e1-daf9-4dd5-af20-c56b4f72d818 None None] Releasing lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 19 01:43:34.474894 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server [None req-b85036e1-daf9-4dd5-af20-c56b4f72d818 None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 19 01:43:34.474894 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 19 01:43:34.474894 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 19 01:43:34.474894 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 19 01:43:34.474894 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 19 01:43:34.474894 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 19 01:43:34.474894 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 19 01:43:34.474894 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 19 01:43:34.474894 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 19 01:43:34.474894 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 19 01:43:34.474894 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 19 01:43:34.474894 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 19 01:43:34.474894 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 19 01:43:34.474894 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 19 01:43:34.474894 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 19 01:43:34.474894 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 19 01:43:34.474894 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 19 01:43:34.474894 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 19 01:43:34.474894 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 19 01:43:34.474894 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 19 01:43:34.474894 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 19 01:43:34.474894 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 19 01:43:34.474894 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 19 01:43:34.474894 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 19 01:43:34.474894 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 19 01:43:35.739148 n-d-853704-1 nova-conductor[120657]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 19 01:43:44.384761 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=11,nova_api:UPDATE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:43:44.386654 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=13,nova_api:DELETE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:43:44.469757 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:43:44.470541 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell1:SELECT=11,nova_cell1:INSERT=17 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:43:54.288241 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=3,nova_cell0:UPDATE=2 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:44:04.319568 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=15,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:44:34.293218 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:44:44.325343 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=7,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:45:14.301115 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:45:24.336170 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:45:34.310925 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=3,nova_cell0:UPDATE=2 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:45:54.313845 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=4,nova_cell0:UPDATE=2 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:45:54.331404 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=6,nova_cell0:UPDATE=3 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:46:13.348247 n-d-853704-1 nova-conductor[120657]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 19 01:46:14.320003 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=4,nova_cell0:UPDATE=2 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:46:34.335793 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:46:41.638948 n-d-853704-1 nova-conductor[120658]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 19 01:46:54.324339 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:46:55.748156 n-d-853704-1 nova-conductor[120657]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 19 01:47:04.338880 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=5,nova_cell0:UPDATE=3 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:47:34.327068 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:47:44.334497 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:47:54.329123 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=3,nova_cell0:UPDATE=2 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:48:14.338730 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=5,nova_cell0:UPDATE=3 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:48:14.343150 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=4,nova_cell0:UPDATE=2 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:48:54.339869 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:48:54.342681 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:49:24.346261 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=5,nova_cell0:UPDATE=3 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:49:33.359481 n-d-853704-1 nova-conductor[120657]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 19 01:49:34.340914 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:49:44.356848 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=4,nova_cell0:UPDATE=2 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:49:56.520375 n-d-853704-1 nova-conductor[120658]: ERROR oslo.messaging._drivers.impl_rabbit [None req-dc1076cf-2a47-496c-acbc-d33e626a1587 tempest-TestShelveInstance-1546100353 tempest-TestShelveInstance-1546100353-project] [fe4e6e0b-b1ce-4fa1-b3d0-5231d7497da8] AMQP server on 10.222.0.49:5672 is unreachable: [Errno 104] Connection reset by peer. Trying again in 1 seconds.: ConnectionResetError: [Errno 104] Connection reset by peer Aug 19 01:49:57.527783 n-d-853704-1 nova-conductor[120658]: INFO oslo.messaging._drivers.impl_rabbit [None req-dc1076cf-2a47-496c-acbc-d33e626a1587 tempest-TestShelveInstance-1546100353 tempest-TestShelveInstance-1546100353-project] [fe4e6e0b-b1ce-4fa1-b3d0-5231d7497da8] Reconnected to AMQP server on 10.222.0.49:5672 via [amqp] client with port 59712. Aug 19 01:50:00.192593 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-dc1076cf-2a47-496c-acbc-d33e626a1587 tempest-TestShelveInstance-1546100353 tempest-TestShelveInstance-1546100353-project] Took 3.68 seconds to select destinations for 1 instance(s). {{(pid=120658) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 19 01:50:00.213968 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-dc1076cf-2a47-496c-acbc-d33e626a1587 tempest-TestShelveInstance-1546100353 tempest-TestShelveInstance-1546100353-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 01:50:00.214415 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-dc1076cf-2a47-496c-acbc-d33e626a1587 tempest-TestShelveInstance-1546100353 tempest-TestShelveInstance-1546100353-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 01:50:00.214806 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-dc1076cf-2a47-496c-acbc-d33e626a1587 tempest-TestShelveInstance-1546100353 tempest-TestShelveInstance-1546100353-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 01:50:00.406914 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-dc1076cf-2a47-496c-acbc-d33e626a1587 tempest-TestShelveInstance-1546100353 tempest-TestShelveInstance-1546100353-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 01:50:00.407299 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-dc1076cf-2a47-496c-acbc-d33e626a1587 tempest-TestShelveInstance-1546100353 tempest-TestShelveInstance-1546100353-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 01:50:00.407716 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-dc1076cf-2a47-496c-acbc-d33e626a1587 tempest-TestShelveInstance-1546100353 tempest-TestShelveInstance-1546100353-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 01:50:00.408407 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-dc1076cf-2a47-496c-acbc-d33e626a1587 tempest-TestShelveInstance-1546100353 tempest-TestShelveInstance-1546100353-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 01:50:00.408817 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-dc1076cf-2a47-496c-acbc-d33e626a1587 tempest-TestShelveInstance-1546100353 tempest-TestShelveInstance-1546100353-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 01:50:00.409199 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-dc1076cf-2a47-496c-acbc-d33e626a1587 tempest-TestShelveInstance-1546100353 tempest-TestShelveInstance-1546100353-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 01:50:00.424562 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-dc1076cf-2a47-496c-acbc-d33e626a1587 tempest-TestShelveInstance-1546100353 tempest-TestShelveInstance-1546100353-project] Getting quotas for project db0db0be3de94dd0b2c97aa90ec4aebb. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 19 01:50:00.428944 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-dc1076cf-2a47-496c-acbc-d33e626a1587 tempest-TestShelveInstance-1546100353 tempest-TestShelveInstance-1546100353-project] Getting quotas for user ed8ee58026934e589dab5c2ba6328c5f and project db0db0be3de94dd0b2c97aa90ec4aebb. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 19 01:50:00.440027 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-dc1076cf-2a47-496c-acbc-d33e626a1587 tempest-TestShelveInstance-1546100353 tempest-TestShelveInstance-1546100353-project] [instance: 91ce9f16-60ee-45c5-a6d8-af077909339f] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120658) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 19 01:50:00.440808 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-dc1076cf-2a47-496c-acbc-d33e626a1587 tempest-TestShelveInstance-1546100353 tempest-TestShelveInstance-1546100353-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 01:50:00.441244 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-dc1076cf-2a47-496c-acbc-d33e626a1587 tempest-TestShelveInstance-1546100353 tempest-TestShelveInstance-1546100353-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 01:50:00.441631 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-dc1076cf-2a47-496c-acbc-d33e626a1587 tempest-TestShelveInstance-1546100353 tempest-TestShelveInstance-1546100353-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 01:50:00.545111 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-dc1076cf-2a47-496c-acbc-d33e626a1587 tempest-TestShelveInstance-1546100353 tempest-TestShelveInstance-1546100353-project] [instance: 91ce9f16-60ee-45c5-a6d8-af077909339f] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='87b0fec9-a948-41f9-a3c0-f0a6a5a8f1e7',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120658) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 19 01:50:00.546373 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-dc1076cf-2a47-496c-acbc-d33e626a1587 tempest-TestShelveInstance-1546100353 tempest-TestShelveInstance-1546100353-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 01:50:00.546817 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-dc1076cf-2a47-496c-acbc-d33e626a1587 tempest-TestShelveInstance-1546100353 tempest-TestShelveInstance-1546100353-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 01:50:00.547213 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-dc1076cf-2a47-496c-acbc-d33e626a1587 tempest-TestShelveInstance-1546100353 tempest-TestShelveInstance-1546100353-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 01:50:00.778186 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-dc1076cf-2a47-496c-acbc-d33e626a1587 tempest-TestShelveInstance-1546100353 tempest-TestShelveInstance-1546100353-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 01:50:00.778544 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-dc1076cf-2a47-496c-acbc-d33e626a1587 tempest-TestShelveInstance-1546100353 tempest-TestShelveInstance-1546100353-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 01:50:00.778934 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-dc1076cf-2a47-496c-acbc-d33e626a1587 tempest-TestShelveInstance-1546100353 tempest-TestShelveInstance-1546100353-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 01:50:00.779375 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-dc1076cf-2a47-496c-acbc-d33e626a1587 tempest-TestShelveInstance-1546100353 tempest-TestShelveInstance-1546100353-project] Acquiring lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 19 01:50:00.779747 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-dc1076cf-2a47-496c-acbc-d33e626a1587 tempest-TestShelveInstance-1546100353 tempest-TestShelveInstance-1546100353-project] Acquired lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 19 01:50:00.780484 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-65255885-2e78-48b8-9f0a-757f8ea2f375 None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 01:50:00.780888 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-65255885-2e78-48b8-9f0a-757f8ea2f375 None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 01:50:00.781266 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-65255885-2e78-48b8-9f0a-757f8ea2f375 None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 01:50:00.781989 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-65255885-2e78-48b8-9f0a-757f8ea2f375 None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 01:50:00.782467 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-65255885-2e78-48b8-9f0a-757f8ea2f375 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 01:50:00.782842 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-65255885-2e78-48b8-9f0a-757f8ea2f375 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 01:50:00.790836 n-d-853704-1 nova-conductor[120658]: ERROR nova.compute.rpcapi [None req-65255885-2e78-48b8-9f0a-757f8ea2f375 None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 19 01:50:00.791214 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-65255885-2e78-48b8-9f0a-757f8ea2f375 None None] Releasing lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 19 01:50:00.793072 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server [None req-65255885-2e78-48b8-9f0a-757f8ea2f375 None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 19 01:50:00.793072 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 19 01:50:00.793072 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 19 01:50:00.793072 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 19 01:50:00.793072 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 19 01:50:00.793072 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 19 01:50:00.793072 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 19 01:50:00.793072 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 19 01:50:00.793072 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 19 01:50:00.793072 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 19 01:50:00.793072 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 19 01:50:00.793072 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 19 01:50:00.793072 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 19 01:50:00.793072 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 19 01:50:00.793072 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 19 01:50:00.793072 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 19 01:50:00.793072 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 19 01:50:00.793072 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 19 01:50:00.793072 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 19 01:50:00.793072 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 19 01:50:00.793072 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 19 01:50:00.793072 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 19 01:50:00.793072 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 19 01:50:00.793072 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 19 01:50:00.793072 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 19 01:50:10.717061 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=12 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:50:10.717834 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=12,nova_api:UPDATE=1,nova_api:DELETE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:50:10.789514 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:50:10.790128 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell1:SELECT=11,nova_cell1:SAVEPOINT=1,nova_cell1:INSERT=18,nova_cell1:RELEASE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:50:14.346562 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:50:15.756182 n-d-853704-1 nova-conductor[120657]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 19 01:50:24.354404 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=15,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:50:54.339591 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:51:04.356013 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=7,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:51:34.353213 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:51:44.362883 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:51:54.354460 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=3,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:52:24.363448 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:52:34.348666 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:52:53.365910 n-d-853704-1 nova-conductor[120657]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 19 01:53:04.352516 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=5,nova_cell0:UPDATE=3 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:53:04.363674 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=7,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:53:24.357831 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=4,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:53:24.369598 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=4,nova_cell0:UPDATE=2 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:53:35.764294 n-d-853704-1 nova-conductor[120657]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 19 01:54:04.374056 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:54:04.387127 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:54:24.384778 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=4,nova_cell0:UPDATE=2 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:54:44.366583 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:54:44.398403 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=4,nova_cell0:UPDATE=2 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:55:01.651657 n-d-853704-1 nova-conductor[120658]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 19 01:55:24.375921 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:55:24.396942 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:56:04.378921 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:56:04.400387 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:56:13.374556 n-d-853704-1 nova-conductor[120657]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 19 01:56:24.915566 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-ffffd80a-5367-4edd-b117-fd83a668c3ef tempest-TestShelveInstance-1546100353 tempest-TestShelveInstance-1546100353-project] Took 0.43 seconds to select destinations for 1 instance(s). {{(pid=120657) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 19 01:56:24.938097 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-ffffd80a-5367-4edd-b117-fd83a668c3ef tempest-TestShelveInstance-1546100353 tempest-TestShelveInstance-1546100353-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 01:56:24.938524 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-ffffd80a-5367-4edd-b117-fd83a668c3ef tempest-TestShelveInstance-1546100353 tempest-TestShelveInstance-1546100353-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 01:56:24.938914 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-ffffd80a-5367-4edd-b117-fd83a668c3ef tempest-TestShelveInstance-1546100353 tempest-TestShelveInstance-1546100353-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 01:56:25.051848 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-ffffd80a-5367-4edd-b117-fd83a668c3ef tempest-TestShelveInstance-1546100353 tempest-TestShelveInstance-1546100353-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 01:56:25.052284 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-ffffd80a-5367-4edd-b117-fd83a668c3ef tempest-TestShelveInstance-1546100353 tempest-TestShelveInstance-1546100353-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 01:56:25.052677 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-ffffd80a-5367-4edd-b117-fd83a668c3ef tempest-TestShelveInstance-1546100353 tempest-TestShelveInstance-1546100353-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 01:56:25.053339 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-ffffd80a-5367-4edd-b117-fd83a668c3ef tempest-TestShelveInstance-1546100353 tempest-TestShelveInstance-1546100353-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 01:56:25.053756 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-ffffd80a-5367-4edd-b117-fd83a668c3ef tempest-TestShelveInstance-1546100353 tempest-TestShelveInstance-1546100353-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 01:56:25.054141 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-ffffd80a-5367-4edd-b117-fd83a668c3ef tempest-TestShelveInstance-1546100353 tempest-TestShelveInstance-1546100353-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 01:56:25.070310 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-ffffd80a-5367-4edd-b117-fd83a668c3ef tempest-TestShelveInstance-1546100353 tempest-TestShelveInstance-1546100353-project] Getting quotas for project db0db0be3de94dd0b2c97aa90ec4aebb. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 19 01:56:25.074789 n-d-853704-1 nova-conductor[120657]: DEBUG nova.quota [None req-ffffd80a-5367-4edd-b117-fd83a668c3ef tempest-TestShelveInstance-1546100353 tempest-TestShelveInstance-1546100353-project] Getting quotas for user ed8ee58026934e589dab5c2ba6328c5f and project db0db0be3de94dd0b2c97aa90ec4aebb. Resources: {'cores', 'instances', 'ram'} {{(pid=120657) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 19 01:56:25.085596 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-ffffd80a-5367-4edd-b117-fd83a668c3ef tempest-TestShelveInstance-1546100353 tempest-TestShelveInstance-1546100353-project] [instance: 172ed085-045f-4483-b746-d0424a57536b] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120657) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 19 01:56:25.086399 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-ffffd80a-5367-4edd-b117-fd83a668c3ef tempest-TestShelveInstance-1546100353 tempest-TestShelveInstance-1546100353-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 01:56:25.086783 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-ffffd80a-5367-4edd-b117-fd83a668c3ef tempest-TestShelveInstance-1546100353 tempest-TestShelveInstance-1546100353-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 01:56:25.087172 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-ffffd80a-5367-4edd-b117-fd83a668c3ef tempest-TestShelveInstance-1546100353 tempest-TestShelveInstance-1546100353-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 01:56:25.149627 n-d-853704-1 nova-conductor[120657]: DEBUG nova.conductor.manager [None req-ffffd80a-5367-4edd-b117-fd83a668c3ef tempest-TestShelveInstance-1546100353 tempest-TestShelveInstance-1546100353-project] [instance: 172ed085-045f-4483-b746-d0424a57536b] block_device_mapping [BlockDeviceMapping(attachment_id=ef821fad-29d5-4cd4-8a47-5f59a4d35c77,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='volume',device_name=None,device_type=None,disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id=None,instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='volume',tag=None,updated_at=,uuid=,volume_id='54a3d5c6-0bba-4f39-80fb-ca159e176b1e',volume_size=1,volume_type=None)] {{(pid=120657) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 19 01:56:25.150903 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-ffffd80a-5367-4edd-b117-fd83a668c3ef tempest-TestShelveInstance-1546100353 tempest-TestShelveInstance-1546100353-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 01:56:25.151343 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-ffffd80a-5367-4edd-b117-fd83a668c3ef tempest-TestShelveInstance-1546100353 tempest-TestShelveInstance-1546100353-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 01:56:25.151772 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-ffffd80a-5367-4edd-b117-fd83a668c3ef tempest-TestShelveInstance-1546100353 tempest-TestShelveInstance-1546100353-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 01:56:25.407261 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-ffffd80a-5367-4edd-b117-fd83a668c3ef tempest-TestShelveInstance-1546100353 tempest-TestShelveInstance-1546100353-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 01:56:25.407730 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-ffffd80a-5367-4edd-b117-fd83a668c3ef tempest-TestShelveInstance-1546100353 tempest-TestShelveInstance-1546100353-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 01:56:25.408629 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-ffffd80a-5367-4edd-b117-fd83a668c3ef tempest-TestShelveInstance-1546100353 tempest-TestShelveInstance-1546100353-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 01:56:25.409177 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-ffffd80a-5367-4edd-b117-fd83a668c3ef tempest-TestShelveInstance-1546100353 tempest-TestShelveInstance-1546100353-project] Acquiring lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 19 01:56:25.409663 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-ffffd80a-5367-4edd-b117-fd83a668c3ef tempest-TestShelveInstance-1546100353 tempest-TestShelveInstance-1546100353-project] Acquired lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 19 01:56:25.410391 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-8bd1cad5-0acc-4cb3-beb4-595d54f2e6ae None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 01:56:25.410802 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-8bd1cad5-0acc-4cb3-beb4-595d54f2e6ae None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 01:56:25.411181 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-8bd1cad5-0acc-4cb3-beb4-595d54f2e6ae None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 01:56:25.411852 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-8bd1cad5-0acc-4cb3-beb4-595d54f2e6ae None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 01:56:25.413836 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-8bd1cad5-0acc-4cb3-beb4-595d54f2e6ae None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 01:56:25.413836 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-8bd1cad5-0acc-4cb3-beb4-595d54f2e6ae None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120657) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 01:56:25.421158 n-d-853704-1 nova-conductor[120657]: ERROR nova.compute.rpcapi [None req-8bd1cad5-0acc-4cb3-beb4-595d54f2e6ae None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 19 01:56:25.421541 n-d-853704-1 nova-conductor[120657]: DEBUG oslo_concurrency.lockutils [None req-8bd1cad5-0acc-4cb3-beb4-595d54f2e6ae None None] Releasing lock "compute-rpcapi-router" {{(pid=120657) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 19 01:56:25.422449 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server [None req-8bd1cad5-0acc-4cb3-beb4-595d54f2e6ae None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 19 01:56:25.422449 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 19 01:56:25.422449 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 19 01:56:25.422449 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 19 01:56:25.422449 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 19 01:56:25.422449 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 19 01:56:25.422449 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 19 01:56:25.422449 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 19 01:56:25.422449 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 19 01:56:25.422449 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 19 01:56:25.422449 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 19 01:56:25.422449 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 19 01:56:25.422449 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 19 01:56:25.422449 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 19 01:56:25.422449 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 19 01:56:25.422449 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 19 01:56:25.422449 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 19 01:56:25.422449 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 19 01:56:25.422449 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 19 01:56:25.422449 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 19 01:56:25.422449 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 19 01:56:25.422449 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 19 01:56:25.422449 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 19 01:56:25.422449 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 19 01:56:25.422449 n-d-853704-1 nova-conductor[120657]: ERROR oslo_messaging.rpc.server Aug 19 01:56:35.228264 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=11,nova_api:UPDATE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:56:35.284041 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_api:SELECT=13,nova_api:DELETE=1 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:56:35.418445 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:56:35.419080 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell1:SELECT=11,nova_cell1:INSERT=16 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:56:44.388876 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=14,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:56:44.402976 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:56:55.772748 n-d-853704-1 nova-conductor[120657]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 19 01:57:24.396757 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:57:24.407125 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:58:04.395759 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:58:04.406488 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:58:21.660868 n-d-853704-1 nova-conductor[120658]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 19 01:58:44.396988 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:58:44.408353 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:59:04.397778 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=3,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:59:04.410360 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=3,nova_cell0:UPDATE=2 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:59:33.385098 n-d-853704-1 nova-conductor[120657]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 19 01:59:44.397712 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 01:59:44.408923 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 02:00:15.781477 n-d-853704-1 nova-conductor[120657]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 19 02:00:24.393362 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 02:00:24.414277 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=7,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 02:00:54.393491 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=5,nova_cell0:UPDATE=3 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 02:00:54.415083 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=6,nova_cell0:UPDATE=3 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 02:01:34.397016 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 02:01:34.418310 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 02:01:41.669571 n-d-853704-1 nova-conductor[120658]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 19 02:01:54.398914 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=4,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 02:01:54.419934 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=4,nova_cell0:UPDATE=2 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 02:02:34.400242 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 02:02:34.420816 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 02:02:51.517486 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-5d80817f-ba3a-4ce8-b36a-f1fc217f9914 tempest-TestSnapshotPattern-401638774 tempest-TestSnapshotPattern-401638774-project] Took 0.66 seconds to select destinations for 1 instance(s). {{(pid=120658) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:911}} Aug 19 02:02:51.539167 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-5d80817f-ba3a-4ce8-b36a-f1fc217f9914 tempest-TestSnapshotPattern-401638774 tempest-TestSnapshotPattern-401638774-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 02:02:51.539592 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-5d80817f-ba3a-4ce8-b36a-f1fc217f9914 tempest-TestSnapshotPattern-401638774 tempest-TestSnapshotPattern-401638774-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 02:02:51.539997 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-5d80817f-ba3a-4ce8-b36a-f1fc217f9914 tempest-TestSnapshotPattern-401638774 tempest-TestSnapshotPattern-401638774-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 02:02:51.775982 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-5d80817f-ba3a-4ce8-b36a-f1fc217f9914 tempest-TestSnapshotPattern-401638774 tempest-TestSnapshotPattern-401638774-project] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 02:02:51.776419 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-5d80817f-ba3a-4ce8-b36a-f1fc217f9914 tempest-TestSnapshotPattern-401638774 tempest-TestSnapshotPattern-401638774-project] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 02:02:51.776812 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-5d80817f-ba3a-4ce8-b36a-f1fc217f9914 tempest-TestSnapshotPattern-401638774 tempest-TestSnapshotPattern-401638774-project] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 02:02:51.777605 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-5d80817f-ba3a-4ce8-b36a-f1fc217f9914 tempest-TestSnapshotPattern-401638774 tempest-TestSnapshotPattern-401638774-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 02:02:51.778019 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-5d80817f-ba3a-4ce8-b36a-f1fc217f9914 tempest-TestSnapshotPattern-401638774 tempest-TestSnapshotPattern-401638774-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 02:02:51.778396 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-5d80817f-ba3a-4ce8-b36a-f1fc217f9914 tempest-TestSnapshotPattern-401638774 tempest-TestSnapshotPattern-401638774-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 02:02:51.797130 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-5d80817f-ba3a-4ce8-b36a-f1fc217f9914 tempest-TestSnapshotPattern-401638774 tempest-TestSnapshotPattern-401638774-project] Getting quotas for project 86520640e48d4aa683e6545deb065edb. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:393}} Aug 19 02:02:51.801752 n-d-853704-1 nova-conductor[120658]: DEBUG nova.quota [None req-5d80817f-ba3a-4ce8-b36a-f1fc217f9914 tempest-TestSnapshotPattern-401638774 tempest-TestSnapshotPattern-401638774-project] Getting quotas for user 20b24a8dbeee41fcbce56eeb59e4be85 and project 86520640e48d4aa683e6545deb065edb. Resources: {'cores', 'instances', 'ram'} {{(pid=120658) _get_quotas /opt/stack/nova/nova/quota.py:383}} Aug 19 02:02:51.813256 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-5d80817f-ba3a-4ce8-b36a-f1fc217f9914 tempest-TestSnapshotPattern-401638774 tempest-TestSnapshotPattern-401638774-project] [instance: 95bdf0ca-bc06-4956-9ff9-5f1523903693] Selected host: n-h2-853704-1; Selected node: n-h2-853704-1; Alternates: [('n-h1-853704-1', 'n-h1-853704-1')] {{(pid=120658) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1666}} Aug 19 02:02:51.814018 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-5d80817f-ba3a-4ce8-b36a-f1fc217f9914 tempest-TestSnapshotPattern-401638774 tempest-TestSnapshotPattern-401638774-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 02:02:51.814467 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-5d80817f-ba3a-4ce8-b36a-f1fc217f9914 tempest-TestSnapshotPattern-401638774 tempest-TestSnapshotPattern-401638774-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 02:02:51.814852 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-5d80817f-ba3a-4ce8-b36a-f1fc217f9914 tempest-TestSnapshotPattern-401638774 tempest-TestSnapshotPattern-401638774-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 02:02:51.876069 n-d-853704-1 nova-conductor[120658]: DEBUG nova.conductor.manager [None req-5d80817f-ba3a-4ce8-b36a-f1fc217f9914 tempest-TestSnapshotPattern-401638774 tempest-TestSnapshotPattern-401638774-project] [instance: 95bdf0ca-bc06-4956-9ff9-5f1523903693] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='87b0fec9-a948-41f9-a3c0-f0a6a5a8f1e7',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=120658) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1406}} Aug 19 02:02:51.877300 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-5d80817f-ba3a-4ce8-b36a-f1fc217f9914 tempest-TestSnapshotPattern-401638774 tempest-TestSnapshotPattern-401638774-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 02:02:51.877835 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-5d80817f-ba3a-4ce8-b36a-f1fc217f9914 tempest-TestSnapshotPattern-401638774 tempest-TestSnapshotPattern-401638774-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 02:02:51.878231 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-5d80817f-ba3a-4ce8-b36a-f1fc217f9914 tempest-TestSnapshotPattern-401638774 tempest-TestSnapshotPattern-401638774-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 02:02:52.177656 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-5d80817f-ba3a-4ce8-b36a-f1fc217f9914 tempest-TestSnapshotPattern-401638774 tempest-TestSnapshotPattern-401638774-project] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 02:02:52.178089 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-5d80817f-ba3a-4ce8-b36a-f1fc217f9914 tempest-TestSnapshotPattern-401638774 tempest-TestSnapshotPattern-401638774-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 02:02:52.178619 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-5d80817f-ba3a-4ce8-b36a-f1fc217f9914 tempest-TestSnapshotPattern-401638774 tempest-TestSnapshotPattern-401638774-project] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 02:02:52.179060 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-5d80817f-ba3a-4ce8-b36a-f1fc217f9914 tempest-TestSnapshotPattern-401638774 tempest-TestSnapshotPattern-401638774-project] Acquiring lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:293}} Aug 19 02:02:52.179401 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-5d80817f-ba3a-4ce8-b36a-f1fc217f9914 tempest-TestSnapshotPattern-401638774 tempest-TestSnapshotPattern-401638774-project] Acquired lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:296}} Aug 19 02:02:52.180168 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-34c07697-e69e-4924-a969-1ac4b5903819 None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 02:02:52.180572 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-34c07697-e69e-4924-a969-1ac4b5903819 None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 02:02:52.180944 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-34c07697-e69e-4924-a969-1ac4b5903819 None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 02:02:52.181584 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-34c07697-e69e-4924-a969-1ac4b5903819 None None] Acquiring lock "33319d98-cfe4-4ac2-b757-392710e3e602" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:385}} Aug 19 02:02:52.181979 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-34c07697-e69e-4924-a969-1ac4b5903819 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:390}} Aug 19 02:02:52.182350 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-34c07697-e69e-4924-a969-1ac4b5903819 None None] Lock "33319d98-cfe4-4ac2-b757-392710e3e602" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=120658) inner /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:404}} Aug 19 02:02:52.191115 n-d-853704-1 nova-conductor[120658]: ERROR nova.compute.rpcapi [None req-34c07697-e69e-4924-a969-1ac4b5903819 None None] Failed to extract compute RPC version from service history because I am too old (minimum version is now 62): IndexError: tuple index out of range Aug 19 02:02:52.191493 n-d-853704-1 nova-conductor[120658]: DEBUG oslo_concurrency.lockutils [None req-34c07697-e69e-4924-a969-1ac4b5903819 None None] Releasing lock "compute-rpcapi-router" {{(pid=120658) lock /usr/local/lib/python3.8/dist-packages/oslo_concurrency/lockutils.py:314}} Aug 19 02:02:52.192467 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server [None req-34c07697-e69e-4924-a969-1ac4b5903819 None None] Exception during message handling: nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 19 02:02:52.192467 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 19 02:02:52.192467 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 520, in _determine_version_cap Aug 19 02:02:52.192467 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = history[service_version]['compute_rpc'] Aug 19 02:02:52.192467 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server IndexError: tuple index out of range Aug 19 02:02:52.192467 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 19 02:02:52.192467 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: Aug 19 02:02:52.192467 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 19 02:02:52.192467 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): Aug 19 02:02:52.192467 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming Aug 19 02:02:52.192467 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) Aug 19 02:02:52.192467 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch Aug 19 02:02:52.192467 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) Aug 19 02:02:52.192467 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch Aug 19 02:02:52.192467 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) Aug 19 02:02:52.192467 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/conductor/manager.py", line 1733, in schedule_and_build_instances Aug 19 02:02:52.192467 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server self.compute_rpcapi.build_and_run_instance( Aug 19 02:02:52.192467 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 1480, in build_and_run_instance Aug 19 02:02:52.192467 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server client = self.router.client(ctxt) Aug 19 02:02:52.192467 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 442, in router Aug 19 02:02:52.192467 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server version_cap = self._determine_version_cap(target) Aug 19 02:02:52.192467 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/rpcapi.py", line 526, in _determine_version_cap Aug 19 02:02:52.192467 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server raise exception.ServiceTooOld(thisver=service_obj.SERVICE_VERSION, Aug 19 02:02:52.192467 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server nova.exception.ServiceTooOld: This service is older (v62) than the minimum (v62) version of the rest of the deployment. Unable to continue. Aug 19 02:02:52.192467 n-d-853704-1 nova-conductor[120658]: ERROR oslo_messaging.rpc.server Aug 19 02:02:53.393412 n-d-853704-1 nova-conductor[120657]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 19 02:03:01.987199 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=17 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 02:03:02.088602 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_api:SELECT=7,nova_api:UPDATE=1,nova_api:DELETE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 02:03:02.188108 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 02:03:02.189820 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell1:SELECT=11,nova_cell1:SAVEPOINT=1,nova_cell1:INSERT=18,nova_cell1:RELEASE=1 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 02:03:14.396508 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 02:03:14.419671 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=14,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 02:03:35.789699 n-d-853704-1 nova-conductor[120657]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 19 02:03:54.392060 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 02:03:54.412685 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 02:04:24.392811 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=5,nova_cell0:UPDATE=3 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 02:04:34.418114 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 02:05:04.392290 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 02:05:14.417961 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 02:05:44.386594 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 02:05:54.405456 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 02:06:04.387792 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=3,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 02:06:13.401565 n-d-853704-1 nova-conductor[120657]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 19 02:06:34.390548 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=6,nova_cell0:UPDATE=3 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 02:06:34.411794 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 02:06:55.797802 n-d-853704-1 nova-conductor[120657]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 19 02:07:14.393809 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 02:07:14.416144 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=7,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 02:07:54.387795 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 02:07:54.410294 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 02:08:21.682382 n-d-853704-1 nova-conductor[120658]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 19 02:08:24.393336 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=5,nova_cell0:UPDATE=3 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 02:08:24.414267 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=5,nova_cell0:UPDATE=3 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 02:09:04.390973 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 02:09:04.410868 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 02:09:24.398257 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=3,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 02:09:33.412252 n-d-853704-1 nova-conductor[120657]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 19 02:09:34.424817 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=5,nova_cell0:UPDATE=3 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 02:10:04.405192 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 02:10:14.430461 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 02:10:15.805867 n-d-853704-1 nova-conductor[120657]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 19 02:10:44.404886 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 02:10:54.427390 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 02:11:24.399937 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 02:11:34.417559 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 02:11:41.690319 n-d-853704-1 nova-conductor[120658]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 19 02:11:44.402887 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=3,nova_cell0:UPDATE=2 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 02:12:14.405984 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 02:12:24.389740 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=9,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 02:12:53.420153 n-d-853704-1 nova-conductor[120657]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 19 02:12:54.407867 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 02:13:04.385944 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 02:13:34.405621 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 02:13:35.814248 n-d-853704-1 nova-conductor[120657]: INFO oslo.messaging._drivers.impl_rabbit [-] A recoverable connection/channel error occurred, trying to reconnect: [Errno 104] Connection reset by peer Aug 19 02:13:44.385945 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=8,nova_cell0:UPDATE=4 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 02:14:04.410990 n-d-853704-1 nova-conductor[120658]: DEBUG dbcounter [-] [120658] Writing DB stats nova_cell0:SELECT=5,nova_cell0:UPDATE=3 {{(pid=120658) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}} Aug 19 02:14:14.392884 n-d-853704-1 nova-conductor[120657]: DEBUG dbcounter [-] [120657] Writing DB stats nova_cell0:SELECT=5,nova_cell0:UPDATE=3 {{(pid=120657) stat_writer /usr/local/lib/python3.8/dist-packages/dbcounter.py:114}}